diff options
474 files changed, 17134 insertions, 11557 deletions
@@ -1 +1,2 @@ *.pyc +*.pyo diff --git a/integration_with_portage.patch b/integration_with_portage.patch index a74beed..ff8f487 100644 --- a/integration_with_portage.patch +++ b/integration_with_portage.patch @@ -1,17 +1,16 @@ -diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py -index 0144cfc..1c423a3 100644 ---- a/pym/_emerge/EbuildBuild.py -+++ b/pym/_emerge/EbuildBuild.py -@@ -9,6 +9,8 @@ from _emerge.CompositeTask import CompositeTask +diff -urN /usr/lib/portage/pym/_emerge/EbuildBuild.py ./pym/_emerge/EbuildBuild.py +--- /usr/lib/portage/pym/_emerge/EbuildBuild.py 2014-01-23 14:09:03.647948532 +0600 ++++ ./pym/_emerge/EbuildBuild.py 2014-02-17 17:25:59.335376529 +0600 +@@ -9,6 +9,8 @@ from _emerge.EbuildMerge import EbuildMerge from _emerge.EbuildFetchonly import EbuildFetchonly from _emerge.EbuildBuildDir import EbuildBuildDir +from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator +from _emerge.EventsLogger import EventsLogger from _emerge.MiscFunctionsProcess import MiscFunctionsProcess - from portage.util import writemsg - import portage -@@ -21,7 +23,7 @@ from portage.package.ebuild._spawn_nofetch import spawn_nofetch + from _emerge.TaskSequence import TaskSequence + +@@ -24,7 +26,7 @@ class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", @@ -20,7 +19,7 @@ index 0144cfc..1c423a3 100644 "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") -@@ -244,8 +246,54 @@ class EbuildBuild(CompositeTask): +@@ -259,8 +261,52 @@ build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) @@ -35,7 +34,7 @@ index 0144cfc..1c423a3 100644 + "depcheckstrict" in self.settings["FEATURES"]: + # Lets start a log listening server + temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) -+ ++ + if "depcheckstrict" not in self.settings["FEATURES"]: + # use default filter_proc + self.logserver=EventsLogger(socket_dir=temp_path) @@ -44,11 +43,11 @@ index 0144cfc..1c423a3 100644 + "This may take some time\n") + filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) + filter_proc=filter_gen.get_filter_proc() -+ self.logserver=EventsLogger(socket_dir=temp_path, ++ self.logserver=EventsLogger(socket_dir=temp_path, + filter_proc=filter_proc) -+ ++ + self.logserver.start() -+ ++ + # Copy socket path to LOG_SOCKET environment variable + env=self.settings.configdict["pkg"] + env['LOG_SOCKET'] = self.logserver.socket_name @@ -62,39 +61,34 @@ index 0144cfc..1c423a3 100644 + env=self.settings.configdict["pkg"] + if 'LOG_SOCKET' in env: + del env['LOG_SOCKET'] -+ ++ + events=self.logserver.stop() + self.logserver=None + analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) + analyser.display() # show the analyse + + #import pdb; pdb.set_trace() -+ -+ + def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default -diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py -index f53570a..82c165d 100644 ---- a/pym/_emerge/EbuildPhase.py -+++ b/pym/_emerge/EbuildPhase.py -@@ -33,7 +33,8 @@ class EbuildPhase(CompositeTask): - ("_ebuild_lock",) +diff -urN /usr/lib/portage/pym/_emerge/EbuildPhase.py ./pym/_emerge/EbuildPhase.py +--- /usr/lib/portage/pym/_emerge/EbuildPhase.py 2014-01-23 14:09:03.648948532 +0600 ++++ ./pym/_emerge/EbuildPhase.py 2014-02-17 17:27:57.509371559 +0600 +@@ -44,7 +44,8 @@ # FEATURES displayed prior to setup phase -- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", -+ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", -+ "distcc-pump", "fakeroot", + _features_display = ( +- "ccache", "compressdebug", "distcc", "distcc-pump", "fakeroot", ++ "ccache", "compressdebug", "depcheck", "depcheckstrict", ++ "distcc", "distcc-pump", "fakeroot", "installsources", "keeptemp", "keepwork", "nostrip", "preserve-libs", "sandbox", "selinux", "sesandbox", "splitdebug", "suidctl", "test", "userpriv", -diff --git a/pym/_emerge/EventsAnalyser.py b/pym/_emerge/EventsAnalyser.py -new file mode 100644 -index 0000000..65ece7b ---- /dev/null -+++ b/pym/_emerge/EventsAnalyser.py -@@ -0,0 +1,511 @@ +diff -urN /usr/lib/portage/pym/_emerge/EventsAnalyser.py ./pym/_emerge/EventsAnalyser.py +--- /usr/lib/portage/pym/_emerge/EventsAnalyser.py 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/_emerge/EventsAnalyser.py 2014-02-17 17:34:27.954355139 +0600 +@@ -0,0 +1,514 @@ +# Distributed under the terms of the GNU General Public License v2 + +import portage @@ -131,36 +125,36 @@ index 0000000..65ece7b + def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): + """ + Gets current dependencies of a package. Looks in portage db -+ ++ + :param pkg: name of package -+ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + ["RDEPEND", "DEPEND"] + :returns: **set** of packages names + """ + ret=set() -+ -+ pkg = self.get_best_visible_pkg(pkg) ++ ++ pkg = self.get_best_visible_pkg(pkg) + if not pkg: + return ret -+ ++ + # we found the best visible match in common tree + + -+ metadata = dict(zip(self.metadata_keys, ++ metadata = dict(zip(self.metadata_keys, + self.portdbapi.aux_get(pkg, self.metadata_keys))) + dep_str = " ".join(metadata[k] for k in dep_type) + + # the IUSE default are very important for us + iuse_defaults=[ + u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] -+ ++ + use=self.use.split() -+ ++ + for u in iuse_defaults: + if u not in use: + use.append(u) + -+ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ success, atoms = portage.dep_check(dep_str, None, self.settings, + myuse=use, myroot=self.settings["ROOT"], + trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) + if not success: @@ -171,7 +165,7 @@ index 0000000..65ece7b + + if not atomname: + continue -+ ++ + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): + for pkg in self.vartree.dep_match(unvirt_pkg): + ret.add(pkg) @@ -180,12 +174,12 @@ index 0000000..65ece7b + + # recursive dependency getter + def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): -+ """ -+ Gets current dependencies of a package on any depth ++ """ ++ Gets current dependencies of a package on any depth + All dependencies **must** be installed -+ ++ + :param pkg: name of package -+ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + ["RDEPEND", "DEPEND"] + :returns: **set** of packages names + """ @@ -194,14 +188,14 @@ index 0000000..65ece7b + + # get porttree dependencies on the first package + -+ pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) ++ pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) + if not pkg: + return ret + + known_packages=set() + unknown_packages=self.get_dep(pkg,dep_type) + ret=ret.union(unknown_packages) -+ ++ + while unknown_packages: + p=unknown_packages.pop() + if p in known_packages: @@ -211,18 +205,18 @@ index 0000000..65ece7b + metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) + + dep_str = " ".join(metadata[k] for k in dep_type) -+ ++ + # the IUSE default are very important for us + iuse_defaults=[ + u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] -+ ++ + use=self.use.split() -+ ++ + for u in iuse_defaults: + if u not in use: + use.append(u) -+ -+ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ ++ success, atoms = portage.dep_check(dep_str, None, self.settings, + myuse=use, myroot=self.settings["ROOT"], + trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) + @@ -233,7 +227,7 @@ index 0000000..65ece7b + atomname = self.vartree.dep_bestmatch(atom) + if not atomname: + continue -+ ++ + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): + for pkg in self.vartree.dep_match(unvirt_pkg): + ret.add(pkg) @@ -241,8 +235,8 @@ index 0000000..65ece7b + return ret + + def get_deps_for_package_building(self, pkg): -+ """ -+ returns buildtime dependencies of current package and ++ """ ++ returns buildtime dependencies of current package and + all runtime dependencies of that buildtime dependencies + """ + buildtime_deps=self.get_dep(pkg, ["DEPEND"]) @@ -254,9 +248,9 @@ index 0000000..65ece7b + return ret + + def get_system_packages_list(self): -+ """ ++ """ + returns all packages from system set. They are always implicit dependencies -+ ++ + :returns: **list** of package names + """ + ret=[] @@ -269,11 +263,12 @@ index 0000000..65ece7b + + +class GentoolkitUtils: -+ """ -+ Interface with qfile and qlist utils. They are much faster than ++ """ ++ Interface with qfile and qlist utils. They are much faster than + internals. + """ + ++ @staticmethod + def getpackagesbyfiles(files): + """ + :param files: list of filenames @@ -287,14 +282,14 @@ index 0000000..65ece7b + ret[f]="directory" + else: + listtocheck.append(f) -+ ++ + try: + proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], -+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, + bufsize=4096) -+ ++ + out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) -+ ++ + lines=out.decode("utf8").split("\n") + #print lines + line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") @@ -309,24 +304,25 @@ index 0000000..65ece7b + + except OSError as e: + portage.util.writemsg("Error while launching qfile: %s\n" % e) -+ -+ ++ ++ + return ret -+ ++ ++ @staticmethod + def getfilesbypackages(packagenames): + """ -+ ++ + :param packagename: name of package + :returns: **list** of files in package with name *packagename* + """ + ret=[] + try: + proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, -+ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, + bufsize=4096) -+ ++ + out,err=proc.communicate() -+ ++ + ret=out.decode("utf8").split("\n") + if ret==['']: + ret=[] @@ -334,34 +330,35 @@ index 0000000..65ece7b + portage.util.writemsg("Error while launching qfile: %s\n" % e) + + return ret -+ -+ def get_all_packages_files(): ++ ++ @staticmethod ++ def get_all_packages_files(): + """ + Memory-hungry operation -+ ++ + :returns: **set** of all files that belongs to package + """ + ret=[] + try: + proc=subprocess.Popen(['qlist']+['--all',"--obj"], -+ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, + bufsize=4096) -+ ++ + out,err=proc.communicate() -+ ++ + ret=out.decode("utf8").split("\n") + except OSError as e: + portage.util.writemsg("Error while launching qfile: %s\n" % e) + + return set(ret) -+ ++ +class FilterProcGenerator: + def __init__(self, pkgname, settings): + portageutils=PortageUtils(settings=settings) + + deps_all=portageutils.get_deps_for_package_building(pkgname) + deps_portage=portageutils.get_dep('portage',["RDEPEND"]) -+ ++ + system_packages=portageutils.get_system_packages_list() + + allfiles=GentoolkitUtils.get_all_packages_files() @@ -369,8 +366,8 @@ index 0000000..65ece7b + "a list of allowed files\n") + + -+ allowedpkgs=system_packages+list(deps_portage)+list(deps_all) -+ ++ allowedpkgs=system_packages+list(deps_portage)+list(deps_all) ++ + allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) + #for pkg in allowedpkgs: + # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) @@ -380,14 +377,14 @@ index 0000000..65ece7b + # manually add all python interpreters to this list + allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) + allowedfiles=set(allowedfiles) -+ ++ + deniedfiles=allfiles-allowedfiles + + def filter_proc(eventname,filename,stage): + if filename in deniedfiles: + return False + return True -+ ++ + self.filter_proc=filter_proc + def get_filter_proc(self): + return self.filter_proc @@ -402,10 +399,10 @@ index 0000000..65ece7b + self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) + self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) + self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) -+ ++ + self.system_packages=self.portageutils.get_system_packages_list() + # All analyse work is here -+ ++ + # get unique filenames + filenames=set() + for stage in events: @@ -416,7 +413,7 @@ index 0000000..65ece7b + filenames=list(filenames) + + file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) -+ # This part is completly unreadable. ++ # This part is completly unreadable. + # It converting one complex struct(returned by getfsevents) to another complex + # struct which good for generating output. + # @@ -427,24 +424,24 @@ index 0000000..65ece7b + for stage in sorted(events): + succ_events=events[stage][0] + fail_events=events[stage][1] -+ ++ + for filename in succ_events: + if filename in file_to_package: + package=file_to_package[filename] + else: + package="unknown" -+ ++ + if not package in packagesinfo: + packagesinfo[package]={} + stageinfo=packagesinfo[package] + if not stage in stageinfo: + stageinfo[stage]={} -+ ++ + filesinfo=stageinfo[stage] + if not filename in filesinfo: + filesinfo[filename]={"found":[],"notfound":[]} + filesinfo[filename]["found"]=succ_events[filename] -+ ++ + for filename in fail_events: + if filename in file_to_package: + package=file_to_package[filename] @@ -455,13 +452,13 @@ index 0000000..65ece7b + stageinfo=packagesinfo[package] + if not stage in stageinfo: + stageinfo[stage]={} -+ ++ + filesinfo=stageinfo[stage] + if not filename in filesinfo: + filesinfo[filename]={"found":[],"notfound":[]} + filesinfo[filename]["notfound"]=fail_events[filename] + self.packagesinfo=packagesinfo -+ ++ + def display(self): + portage.util.writemsg( + portage.output.colorize( @@ -470,12 +467,12 @@ index 0000000..65ece7b + stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, + "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} + packagesinfo=self.packagesinfo -+ # print information grouped by package ++ # print information grouped by package + for package in sorted(packagesinfo): + # not showing special directory package + if package=="directory": + continue -+ ++ + if package=="unknown": + continue + @@ -492,7 +489,7 @@ index 0000000..65ece7b + + if len(stages)==0: + continue -+ ++ + filenames={} + for stage in stages: + for filename in packagesinfo[package][stage]: @@ -503,7 +500,7 @@ index 0000000..65ece7b + else: + status, old_was_readed, old_was_writed=filenames[filename] + filenames[filename]=[ -+ 'ok',old_was_readed | was_readed, old_was_writed | was_writed ++ 'ok',old_was_readed | was_readed, old_was_writed | was_writed + ] + if len(packagesinfo[package][stage][filename]["notfound"])!=0: + was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] @@ -512,9 +509,9 @@ index 0000000..65ece7b + else: + status, old_was_notfound, old_was_blocked=filenames[filename] + filenames[filename]=[ -+ 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked ++ 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked + ] -+ ++ + + if is_pkg_in_dep: + portage.util.writemsg("[OK]") @@ -543,9 +540,9 @@ index 0000000..65ece7b + ('err',False,True):"blocked", + ('err',True,True):"not found and blocked" + } -+ ++ + filescounter=0 -+ ++ + for filename in filenames: + event_info=tuple(filenames[filename]) + portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) @@ -553,7 +550,7 @@ index 0000000..65ece7b + if filescounter>10: + portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) + break -+ # ... and one more check. Making sure that direct build time ++ # ... and one more check. Making sure that direct build time + # dependencies were accessed + #import pdb; pdb.set_trace() + not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) @@ -562,7 +559,7 @@ index 0000000..65ece7b + portage.util.writemsg("Warning! Some build time dependencies " + \ + "of packages were not accessed: " + \ + " ".join(not_accessed_deps) + "\n") -+ ++ + def is_package_useful(self,pkg,stages,files): + """ some basic heuristics here to cut part of packages """ + @@ -596,22 +593,19 @@ index 0000000..65ece7b + for f in files: + if is_file_excluded(f): + continue -+ -+ # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 ++ ++ # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 + if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): + break + else: + return False # we get here if cycle ends not with break -+ ++ + return True -+ -+ -\ No newline at end of file -diff --git a/pym/_emerge/EventsLogger.py b/pym/_emerge/EventsLogger.py -new file mode 100644 -index 0000000..68b3c67 ---- /dev/null -+++ b/pym/_emerge/EventsLogger.py ++ ++ +diff -urN /usr/lib/portage/pym/_emerge/EventsLogger.py ./pym/_emerge/EventsLogger.py +--- /usr/lib/portage/pym/_emerge/EventsLogger.py 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/_emerge/EventsLogger.py 2014-02-17 17:36:42.034349501 +0600 @@ -0,0 +1,180 @@ +# Distributed under the terms of the GNU General Public License v2 + @@ -629,17 +623,17 @@ index 0000000..68b3c67 +class EventsLogger(threading.Thread): + def default_filter(eventname, filename, stage): + return True -+ ++ + def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): + threading.Thread.__init__(self) # init the Thread -+ ++ + self.alive=False -+ ++ + self.main_thread=threading.currentThread() -+ ++ + self.socket_dir=socket_dir + self.filter_proc=filter_proc -+ ++ + self.socket_name=None + self.socket_logger=None + @@ -648,16 +642,16 @@ index 0000000..68b3c67 + try: + socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, + prefix="log_socket_") -+ ++ + socket_name = os.path.join(socket_dir_name, 'socket') + + except OSError as e: + return -+ ++ + self.socket_name=socket_name -+ ++ + #print(self.socket_name) -+ ++ + try: + socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -680,21 +674,21 @@ index 0000000..68b3c67 + stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) + except OSError as e: + return -+ ++ + def run(self): + """ Starts the log server """ + + self.alive=True + self.listen_thread=threading.currentThread() + clients={} -+ ++ + epoll=select.epoll() + epoll.register(self.socket_logger.fileno(), select.EPOLLIN) + + while self.alive: + try: + sock_events = epoll.poll(3) -+ ++ + for fileno, sock_event in sock_events: + if fileno == self.socket_logger.fileno(): + ret = self.socket_logger.accept() @@ -707,13 +701,13 @@ index 0000000..68b3c67 + elif sock_event & select.EPOLLIN: + s=clients[fileno] + record=s.recv(8192) -+ ++ + if not record: # if connection was closed + epoll.unregister(fileno) + clients[fileno].close() + del clients[fileno] + continue -+ ++ + #import pdb; pdb.set_trace() + try: + message=record.decode("utf8").split("\0") @@ -724,7 +718,7 @@ index 0000000..68b3c67 + # continue + + #print(message) -+ ++ + try: + if message[4]=="ASKING": + if self.filter_proc(message[1],message[2],message[3]): @@ -738,7 +732,7 @@ index 0000000..68b3c67 + + if not stage in self.events: + self.events[stage]=[{},{}] -+ ++ + hashofsucesses=self.events[stage][0] + hashoffailures=self.events[stage][1] + @@ -748,19 +742,19 @@ index 0000000..68b3c67 + if result=="OK": + if not filename in hashofsucesses: + hashofsucesses[filename]=[False,False] -+ ++ + readed_or_writed=hashofsucesses[filename] -+ ++ + if eventname=="read": + readed_or_writed[0]=True + elif eventname=="write": + readed_or_writed[1]=True -+ ++ + elif result[0:3]=="ERR" or result=="DENIED": + if not filename in hashoffailures: + hashoffailures[filename]=[False,False] + notfound_or_blocked=hashoffailures[filename] -+ ++ + if result=="ERR/2": + notfound_or_blocked[0]=True + elif result=="DENIED": @@ -768,74 +762,122 @@ index 0000000..68b3c67 + + else: + print("Error in logger module<->analyser protocol") -+ ++ + except IndexError: + print("IndexError while parsing %s" % record) + except IOError as e: + if e.errno!=4: # handling "Interrupted system call" errors + raise -+ -+ # if main thread doesnt exists then exit ++ ++ # if main thread doesnt exists then exit + if not self.main_thread.is_alive(): + break + epoll.unregister(self.socket_logger.fileno()) + epoll.close() + self.socket_logger.close() -+ ++ + def stop(self): + """ Stops the log server. Returns all events """ + + self.alive=False -+ ++ + # Block the main thread until listener exists + self.listen_thread.join() -+ ++ + # We assume portage clears tmp folder, so no deleting a socket file + # We assume that no new socket data will arrive after this moment + return self.events -diff --git a/pym/portage/const.py b/pym/portage/const.py -index ecaa8f1..f34398d 100644 ---- a/pym/portage/const.py -+++ b/pym/portage/const.py -@@ -67,6 +67,8 @@ FAKEROOT_BINARY = "/usr/bin/fakeroot" +diff -urN /usr/lib/portage/pym/portage/const.py ./pym/portage/const.py +--- /usr/lib/portage/pym/portage/const.py 2014-01-23 14:09:04.077948503 +0600 ++++ ./pym/portage/const.py 2014-02-17 17:38:12.197345709 +0600 +@@ -72,6 +72,7 @@ BASH_BINARY = "/bin/bash" MOVE_BINARY = "/bin/mv" PRELINK_BINARY = "/usr/sbin/prelink" -+AUTODEP_LIBRARY = "/usr/lib/file_hook.so" -+ ++AUTODEP_LIBRARY = "/usr/lib/file_hook.so" INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" REPO_NAME_FILE = "repo_name" -@@ -88,7 +90,8 @@ EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure" - SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", -- "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "fixpackages", "force-mirror", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", -diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py -index 87aa606..6d42809 100644 ---- a/pym/portage/package/ebuild/_config/special_env_vars.py -+++ b/pym/portage/package/ebuild/_config/special_env_vars.py -@@ -101,8 +101,8 @@ environ_whitelist += [ +@@ -133,6 +134,8 @@ + "compressdebug", + "compress-index", + "config-protect-if-modified", ++ "depcheck", ++ "depcheckstrict", + "digest", + "distcc", + "distcc-pump", +diff -urN /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py ./pym/portage/package/ebuild/_config/special_env_vars.py +--- /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py 2014-01-23 14:09:06.463948342 +0600 ++++ ./pym/portage/package/ebuild/_config/special_env_vars.py 2014-02-17 17:40:56.800338787 +0600 +@@ -113,7 +113,7 @@ # other variables inherited from the calling environment environ_whitelist += [ "CVS_RSH", "ECHANGELOG_USER", - "GPG_AGENT_INFO", -- "SSH_AGENT_PID", "SSH_AUTH_SOCK", + "GPG_AGENT_INFO", "LOG_SOCKET", -+ "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "SSH_AGENT_PID", "SSH_AUTH_SOCK", "STY", "WINDOW", "XAUTHORITY", ] +diff -urN /usr/lib/portage/pym/portage/package/ebuild/doebuild.py ./pym/portage/package/ebuild/doebuild.py +--- /usr/lib/portage/pym/portage/package/ebuild/doebuild.py 2014-01-23 14:09:06.183948361 +0600 ++++ ./pym/portage/package/ebuild/doebuild.py 2014-02-17 17:43:25.387332538 +0600 +@@ -66,7 +66,7 @@ + from portage.util import apply_recursive_permissions, \ + apply_secpass_permissions, noiselimit, normalize_path, \ + writemsg, writemsg_stdout, write_atomic +-from portage.util.lafilefixer import rewrite_lafile ++from portage.util.lafilefixer import rewrite_lafile + from portage.versions import _pkgsplit + from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor + from _emerge.EbuildBuildDir import EbuildBuildDir +@@ -492,7 +492,7 @@ + """ + Wrapper function that invokes specific ebuild phases through the spawning + of ebuild.sh +- ++ + @param myebuild: name of the ebuild to invoke the phase on (CPV) + @type myebuild: String + @param mydo: Phase to run +@@ -535,13 +535,13 @@ + @return: + 1. 0 for success + 2. 1 for error +- ++ + Most errors have an accompanying error message. +- ++ + listonly and fetchonly are only really necessary for operations involving 'fetch' + prev_mtimes are only necessary for merge operations. + Other variables may not be strictly required, many have defaults that are set inside of doebuild. +- ++ + """ -diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py -index 49b67ac..c76c1ed 100644 ---- a/pym/portage/package/ebuild/doebuild.py -+++ b/pym/portage/package/ebuild/doebuild.py -@@ -1038,6 +1038,9 @@ def _spawn_actionmap(settings): + if settings is None: +@@ -563,8 +563,8 @@ + if not tree: + writemsg("Warning: tree not specified to doebuild\n") + tree = "porttree" +- +- # chunked out deps for each phase, so that ebuild binary can use it ++ ++ # chunked out deps for each phase, so that ebuild binary can use it + # to collapse targets down. + actionmap_deps={ + "pretend" : [], +@@ -579,7 +579,7 @@ + "package":["install"], + "merge" :["install"], + } +- ++ + if mydbapi is None: + mydbapi = portage.db[myroot][tree].dbapi + +@@ -1306,6 +1306,9 @@ nosandbox = ("sandbox" not in features and \ "usersandbox" not in features) @@ -845,7 +887,7 @@ index 49b67ac..c76c1ed 100644 if not portage.process.sandbox_capable: nosandbox = True -@@ -1215,7 +1218,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero +@@ -1551,7 +1554,10 @@ keywords["opt_name"] = "[%s/%s]" % \ (mysettings.get("CATEGORY",""), mysettings.get("PF","")) @@ -857,20 +899,19 @@ index 49b67ac..c76c1ed 100644 keywords["opt_name"] += " bash" spawn_func = portage.process.spawn_bash elif fakeroot: -diff --git a/pym/portage/process.py b/pym/portage/process.py -index 3c15370..6866a2f 100644 ---- a/pym/portage/process.py -+++ b/pym/portage/process.py -@@ -16,7 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:dump_traceback', +diff -urN /usr/lib/portage/pym/portage/process.py ./pym/portage/process.py +--- /usr/lib/portage/pym/portage/process.py 2014-01-23 14:09:04.079948503 +0600 ++++ ./pym/portage/process.py 2014-02-17 17:45:29.526327317 +0600 +@@ -22,7 +22,7 @@ + 'portage.util:dump_traceback,writemsg', ) -from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY +from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY from portage.exception import CommandNotFound + from portage.util._ctypes import find_library, LoadLibrary, ctypes - try: -@@ -39,6 +39,9 @@ else: +@@ -87,13 +87,16 @@ sandbox_capable = (os.path.isfile(SANDBOX_BINARY) and os.access(SANDBOX_BINARY, os.X_OK)) @@ -880,7 +921,15 @@ index 3c15370..6866a2f 100644 fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and os.access(FAKEROOT_BINARY, os.X_OK)) -@@ -66,6 +69,16 @@ def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): + def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): + """ + Spawns a bash shell running a specific commands +- ++ + @param mycommand: The command for bash to run + @type mycommand: String + @param debug: Turn bash debugging on (set -x) +@@ -114,6 +117,16 @@ args.append(mycommand) return spawn(args, opt_name=opt_name, **keywords) @@ -892,8 +941,44 @@ index 3c15370..6866a2f 100644 + + # Core part: tell the loader to preload logging library + keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY -+ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) + def spawn_sandbox(mycommand, opt_name=None, **keywords): if not sandbox_capable: return spawn_bash(mycommand, opt_name=opt_name, **keywords) +@@ -199,7 +212,7 @@ + unshare_ipc=False, cgroup=None): + """ + Spawns a given command. +- ++ + @param mycommand: the command to execute + @type mycommand: String or List (Popen style list) + @param env: A dict of Key=Value pairs for env variables +@@ -238,7 +251,7 @@ + + logfile requires stdout and stderr to be assigned to this process (ie not pointed + somewhere else.) +- ++ + """ + + # mycommand is either a str or a list +@@ -387,7 +400,7 @@ + + """ + Execute a given binary with options +- ++ + @param binary: Name of program to execute + @type binary: String + @param mycommand: Options for program +@@ -627,7 +640,7 @@ + def find_binary(binary): + """ + Given a binary name, find the binary in PATH +- ++ + @param binary: Name of the binary to find + @type string + @rtype: None or string diff --git a/portage_with_autodep/bin/archive-conf b/portage_with_autodep/bin/archive-conf index 7978668..2c34588 100755 --- a/portage_with_autodep/bin/archive-conf +++ b/portage_with_autodep/bin/archive-conf @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # @@ -12,43 +12,21 @@ from __future__ import print_function import sys -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True + +import portage.dispatch_conf from portage import os -from portage import dispatch_conf +from portage.checksum import perform_md5 FIND_EXTANT_CONTENTS = "find %s -name CONTENTS" MANDATORY_OPTS = [ 'archive-dir' ] -try: - import fchksum - def perform_checksum(filename): return fchksum.fmd5t(filename) -except ImportError: - import md5 - def md5_to_hex(md5sum): - hexform = "" - for ix in range(len(md5sum)): - hexform = hexform + "%02x" % ord(md5sum[ix]) - return hexform.lower() - - def perform_checksum(filename): - f = open(filename, 'rb') - blocksize=32768 - data = f.read(blocksize) - size = 0 - sum = md5.new() - while data: - sum.update(data) - size = size + len(data) - data = f.read(blocksize) - return (md5_to_hex(sum.digest()),size) - def archive_conf(): args = [] content_files = [] @@ -63,19 +41,19 @@ def archive_conf(): md5_match_hash[conf] = '' # Find all the CONTENT files in VDB_PATH. - content_files += os.popen(FIND_EXTANT_CONTENTS % - (os.path.join(portage.settings['EROOT'], portage.VDB_PATH))).readlines() + with os.popen(FIND_EXTANT_CONTENTS % (os.path.join(portage.settings['EROOT'], portage.VDB_PATH))) as f: + content_files += f.readlines() # Search for the saved md5 checksum of all the specified config files # and see if the current file is unmodified or not. try: todo_cnt = len(args) - for file in content_files: - file = file.rstrip() + for filename in content_files: + filename = filename.rstrip() try: - contents = open(file, "r") + contents = open(filename, "r") except IOError as e: - print('archive-conf: Unable to open %s: %s' % (file, e), file=sys.stderr) + print('archive-conf: Unable to open %s: %s' % (filename, e), file=sys.stderr) sys.exit(1) lines = contents.readlines() for line in lines: @@ -84,7 +62,7 @@ def archive_conf(): for conf in args: if items[1] == conf: stored = items[2].lower() - real = perform_checksum(conf)[0].lower() + real = perform_md5(conf).lower() if stored == real: md5_match_hash[conf] = conf todo_cnt -= 1 diff --git a/portage_with_autodep/bin/banned-helper b/portage_with_autodep/bin/banned-helper deleted file mode 100755 index 17ea991..0000000 --- a/portage_with_autodep/bin/banned-helper +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright 2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -die "'${0##*/}' has been banned for EAPI '$EAPI'" -exit 1 diff --git a/portage_with_autodep/bin/bashrc-functions.sh b/portage_with_autodep/bin/bashrc-functions.sh index 4da5585..503b172 100755 --- a/portage_with_autodep/bin/bashrc-functions.sh +++ b/portage_with_autodep/bin/bashrc-functions.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 portageq() { - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}}\ "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" "$@" } @@ -23,71 +23,16 @@ register_success_hook() { done } -strip_duplicate_slashes() { +__strip_duplicate_slashes() { if [[ -n $1 ]] ; then local removed=$1 while [[ ${removed} == *//* ]] ; do removed=${removed//\/\///} done - echo ${removed} + echo "${removed}" fi } -# this is a function for removing any directory matching a passed in pattern from -# PATH -remove_path_entry() { - save_IFS - IFS=":" - stripped_path="${PATH}" - while [ -n "$1" ]; do - cur_path="" - for p in ${stripped_path}; do - if [ "${p/${1}}" == "${p}" ]; then - cur_path="${cur_path}:${p}" - fi - done - stripped_path="${cur_path#:*}" - shift - done - restore_IFS - PATH="${stripped_path}" -} - -# Set given variables unless these variable have been already set (e.g. during emerge -# invocation) to values different than values set in make.conf. -set_unless_changed() { - if [[ $# -lt 1 ]]; then - die "${FUNCNAME}() requires at least 1 argument: VARIABLE=VALUE" - fi - - local argument value variable - for argument in "$@"; do - if [[ ${argument} != *=* ]]; then - die "${FUNCNAME}(): Argument '${argument}' has incorrect syntax" - fi - variable="${argument%%=*}" - value="${argument#*=}" - if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then - eval "${variable}=\"\${value}\"" - fi - done -} - -# Unset given variables unless these variable have been set (e.g. during emerge -# invocation) to values different than values set in make.conf. -unset_unless_changed() { - if [[ $# -lt 1 ]]; then - die "${FUNCNAME}() requires at least 1 argument: VARIABLE" - fi - - local variable - for variable in "$@"; do - if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then - unset ${variable} - fi - done -} - KV_major() { [[ -z $1 ]] && return 1 diff --git a/portage_with_autodep/bin/binhost-snapshot b/portage_with_autodep/bin/binhost-snapshot index 9d2697d..376080c 100755 --- a/portage_with_autodep/bin/binhost-snapshot +++ b/portage_with_autodep/bin/binhost-snapshot @@ -1,9 +1,8 @@ #!/usr/bin/python -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io -import optparse import os import sys import textwrap @@ -13,13 +12,12 @@ try: except ImportError: from urlparse import urlparse -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname( - osp.realpath(__file__))), "pym")) - import portage +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True +from portage.util._argparse import ArgumentParser def parse_args(argv): prog_name = os.path.basename(argv[0]) @@ -47,11 +45,12 @@ def parse_args(argv): "write Packages index with\n" + \ " snapshot_uri" - parser = optparse.OptionParser(usage=usage) - parser.add_option('--hardlinks', help='create hardlinks (y or n, default is y)', - choices=('y', 'n')) - parser.set_defaults(hardlinks='y') - options, args = parser.parse_args(argv[1:]) + parser = ArgumentParser(usage=usage) + parser.add_argument('--hardlinks', + help='create hardlinks (y or n, default is y)', + choices=('y', 'n'), + default='y') + options, args = parser.parse_known_args(argv[1:]) if len(args) != 4: parser.error("Required 4 arguments, got %d" % (len(args),)) diff --git a/portage_with_autodep/bin/chpathtool.py b/portage_with_autodep/bin/chpathtool.py index d0d49cb..0cb5d64 100755 --- a/portage_with_autodep/bin/chpathtool.py +++ b/portage_with_autodep/bin/chpathtool.py @@ -1,15 +1,26 @@ #!/usr/bin/python -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +"""Helper tool for converting installed files to custom prefixes. + +In other words, eprefixy $D for Gentoo/Prefix.""" + import io -import optparse import os import stat import sys -CONTENT_ENCODING = "utf_8" -FS_ENCODING = "utf_8" +from portage.util._argparse import ArgumentParser + +# Argument parsing compatibility for Python 2.6 using optparse. +if sys.hexversion < 0x2070000: + from optparse import OptionParser + +from optparse import OptionError + +CONTENT_ENCODING = 'utf_8' +FS_ENCODING = 'utf_8' try: import magic @@ -41,7 +52,9 @@ class IsTextFile(object): def _is_text_magic(self, filename): mime_type = self._m.file(filename) - return mime_type.startswith("text/") + if isinstance(mime_type, bytes): + mime_type = mime_type.decode('ascii', 'replace') + return mime_type.startswith('text/') def _is_text_encoding(self, filename): try: @@ -64,7 +77,7 @@ def chpath_inplace(filename, is_text_file, old, new): try: orig_mode = stat.S_IMODE(os.lstat(filename).st_mode) except OSError as e: - sys.stderr.write("%s: %s\n" % (e, filename)) + sys.stderr.write('%s: %s\n' % (e, filename)) return temp_mode = 0o200 | orig_mode os.chmod(filename, temp_mode) @@ -121,8 +134,12 @@ def chpath_inplace(filename, is_text_file, old, new): f.close() if modified: - orig_mtime = orig_stat[stat.ST_MTIME] - os.utime(filename, (orig_mtime, orig_mtime)) + if sys.hexversion >= 0x3030000: + orig_mtime = orig_stat.st_mtime_ns + os.utime(filename, ns=(orig_mtime, orig_mtime)) + else: + orig_mtime = orig_stat[stat.ST_MTIME] + os.utime(filename, (orig_mtime, orig_mtime)) return modified def chpath_inplace_symlink(filename, st, old, new): @@ -135,14 +152,37 @@ def chpath_inplace_symlink(filename, st, old, new): def main(argv): - usage = "%s [options] <location> <old> <new>" % (os.path.basename(argv[0],)) - parser = optparse.OptionParser(usage=usage) - options, args = parser.parse_args(argv[1:]) - - if len(args) != 3: - parser.error("3 args required, got %s" % (len(args),)) - - location, old, new = args + parser = ArgumentParser(description=__doc__) + try: + parser.add_argument('location', default=None, + help='root directory (e.g. $D)') + parser.add_argument('old', default=None, + help='original build prefix (e.g. /)') + parser.add_argument('new', default=None, + help='new install prefix (e.g. $EPREFIX)') + opts = parser.parse_args(argv) + + location, old, new = opts.location, opts.old, opts.new + except OptionError: + # Argument parsing compatibility for Python 2.6 using optparse. + if sys.hexversion < 0x2070000: + parser = OptionParser(description=__doc__, + usage="usage: %prog [-h] location old new\n\n" + \ + " location: root directory (e.g. $D)\n" + \ + " old: original build prefix (e.g. /)\n" + \ + " new: new install prefix (e.g. $EPREFIX)") + + (opts, args) = parser.parse_args() + + if len(args) != 3: + parser.print_usage() + print("%s: error: expected 3 arguments, got %i" + % (__file__, len(args))) + return + + location, old, new = args[0:3] + else: + raise is_text_file = IsTextFile() @@ -178,5 +218,5 @@ def main(argv): return os.EX_OK -if __name__ == "__main__": - sys.exit(main(sys.argv)) +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/portage_with_autodep/bin/clean_locks b/portage_with_autodep/bin/clean_locks index 8c4299c..184e80c 100755 --- a/portage_with_autodep/bin/clean_locks +++ b/portage_with_autodep/bin/clean_locks @@ -1,21 +1,17 @@ #!/usr/bin/python -O -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import sys, errno -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - -from portage import os +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True if not sys.argv[1:] or "--help" in sys.argv or "-h" in sys.argv: - import portage print() print("You must specify directories with hardlink-locks to clean.") print("You may optionally specify --force, which will remove all") diff --git a/portage_with_autodep/bin/dispatch-conf b/portage_with_autodep/bin/dispatch-conf index 139a001..10455f4 100755 --- a/portage_with_autodep/bin/dispatch-conf +++ b/portage_with_autodep/bin/dispatch-conf @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # @@ -16,19 +16,15 @@ from __future__ import print_function from stat import ST_GID, ST_MODE, ST_UID from random import random import atexit, re, shutil, stat, sys - -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os -from portage import dispatch_conf from portage import _unicode_decode from portage.dispatch_conf import diffstatusoutput -from portage.process import find_binary +from portage.process import find_binary, spawn FIND_EXTANT_CONFIGS = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print" DIFF_CONTENTS = "diff -Nu '%s' '%s'" @@ -83,7 +79,7 @@ class dispatch: confs = [] count = 0 - config_root = portage.const.EPREFIX or os.sep + config_root = portage.settings["EPREFIX"] or os.sep self.options = portage.dispatch_conf.read_config(MANDATORY_OPTS) if "log-file" in self.options: @@ -411,7 +407,8 @@ class dispatch: def do_help (self): - print(); print + print() + print() print(' u -- update current config with new config and continue') print(' z -- zap (delete) new config and continue') @@ -431,7 +428,7 @@ class dispatch: def getch (): # from ASPN - Danny Yoo # - import sys, tty, termios + import tty, termios fd = sys.stdin.fileno() old_settings = termios.tcgetattr(fd) @@ -456,17 +453,18 @@ def clear_screen(): pass os.system("clear 2>/dev/null") -from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") def spawn_shell(cmd): if shell: + sys.__stdout__.flush() + sys.__stderr__.flush() spawn([shell, "-c", cmd], env=os.environ, - fd_pipes = { 0 : sys.stdin.fileno(), - 1 : sys.stdout.fileno(), - 2 : sys.stderr.fileno()}) + fd_pipes = { 0 : portage._get_stdin().fileno(), + 1 : sys.__stdout__.fileno(), + 2 : sys.__stderr__.fileno()}) else: os.system(cmd) diff --git a/portage_with_autodep/bin/dohtml.py b/portage_with_autodep/bin/dohtml.py index f0a7f2c..f98557f 100755 --- a/portage_with_autodep/bin/dohtml.py +++ b/portage_with_autodep/bin/dohtml.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # @@ -11,18 +11,18 @@ # # # Detailed usage: -# dohtml <list-of-files> -# - will install the files in the list of files (space-separated list) into -# /usr/share/doc/${PF}/html, provided the file ends in .htm, .html, .css, -# .js, ,gif, .jpeg, .jpg, or .png. +# dohtml <list-of-files> +# - will install the files in the list of files (space-separated list) into +# /usr/share/doc/${PF}/html, provided the file ends in .css, .gif, .htm, +# .html, .jpeg, .jpg, .js or .png. # dohtml -r <list-of-files-and-directories> -# - will do as 'dohtml', but recurse into all directories, as long as the +# - will do as 'dohtml', but recurse into all directories, as long as the # directory name is not CVS # dohtml -A jpe,java [-r] <list-of-files[-and-directories]> # - will do as 'dohtml' but add .jpe,.java (default filter list is # added to your list) # dohtml -a png,gif,html,htm [-r] <list-of-files[-and-directories]> -# - will do as 'dohtml' but filter on .png,.gif,.html,.htm (default filter +# - will do as 'dohtml' but filter on .png,.gif,.html,.htm (default filter # list is ignored) # dohtml -x CVS,SCCS,RCS -r <list-of-files-and-directories> # - will do as 'dohtml -r', but ignore directories named CVS, SCCS, RCS @@ -31,13 +31,25 @@ from __future__ import print_function import os +import shutil import sys +from portage.util import normalize_path + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + def dodir(path): - os.spawnlp(os.P_WAIT, "install", "install", "-d", path) + try: + os.makedirs(path, 0o755) + except OSError: + if not os.path.isdir(path): + raise + os.chmod(path, 0o755) def dofile(src,dst): - os.spawnlp(os.P_WAIT, "install", "install", "-m0644", src, dst) + shutil.copy(src, dst) + os.chmod(dst, 0o644) def eqawarn(lines): cmd = "source '%s/isolated-functions.sh' ; " % \ @@ -47,32 +59,43 @@ def eqawarn(lines): os.spawnlp(os.P_WAIT, "bash", "bash", "-c", cmd) skipped_directories = [] +skipped_files = [] +warn_on_skipped_files = os.environ.get("PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES") is not None +unwarned_skipped_extensions = os.environ.get("PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", "").split() +unwarned_skipped_files = os.environ.get("PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", "").split() def install(basename, dirname, options, prefix=""): fullpath = basename if prefix: - fullpath = prefix + "/" + fullpath + fullpath = os.path.join(prefix, fullpath) if dirname: - fullpath = dirname + "/" + fullpath + fullpath = os.path.join(dirname, fullpath) if options.DOCDESTTREE: - destdir = options.ED + "usr/share/doc/" + options.PF + "/" + options.DOCDESTTREE + "/" + options.doc_prefix + "/" + prefix + desttree = options.DOCDESTTREE else: - destdir = options.ED + "usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix + desttree = "html" + + destdir = os.path.join(options.ED, "usr", "share", "doc", + options.PF.lstrip(os.sep), desttree.lstrip(os.sep), + options.doc_prefix.lstrip(os.sep), prefix).rstrip(os.sep) if not os.path.exists(fullpath): sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath) return False elif os.path.isfile(fullpath): - ext = os.path.splitext(basename)[1] - if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files: + ext = os.path.splitext(basename)[1][1:] + if ext in options.allowed_exts or basename in options.allowed_files: dodir(destdir) - dofile(fullpath, destdir + "/" + basename) + dofile(fullpath, os.path.join(destdir, basename)) + elif warn_on_skipped_files and ext not in unwarned_skipped_extensions and basename not in unwarned_skipped_files: + skipped_files.append(fullpath) elif options.recurse and os.path.isdir(fullpath) and \ basename not in options.disallowed_dirs: for i in os.listdir(fullpath): pfx = basename - if prefix: pfx = prefix + "/" + pfx + if prefix: + pfx = os.path.join(prefix, pfx) install(i, dirname, options, pfx) elif not options.recurse and os.path.isdir(fullpath): global skipped_directories @@ -88,21 +111,28 @@ class OptionsClass: self.PF = "" self.ED = "" self.DOCDESTTREE = "" - + if "PF" in os.environ: self.PF = os.environ["PF"] + if self.PF: + self.PF = normalize_path(self.PF) if "force-prefix" not in os.environ.get("FEATURES", "").split() and \ os.environ.get("EAPI", "0") in ("0", "1", "2"): self.ED = os.environ.get("D", "") else: self.ED = os.environ.get("ED", "") + if self.ED: + self.ED = normalize_path(self.ED) if "_E_DOCDESTTREE_" in os.environ: self.DOCDESTTREE = os.environ["_E_DOCDESTTREE_"] - - self.allowed_exts = [ 'htm', 'html', 'css', 'js', - 'gif', 'jpeg', 'jpg', 'png' ] + if self.DOCDESTTREE: + self.DOCDESTTREE = normalize_path(self.DOCDESTTREE) + + self.allowed_exts = ['css', 'gif', 'htm', 'html', 'jpeg', 'jpg', 'js', 'png'] + if os.environ.get("EAPI", "0") in ("4-python", "5-progress"): + self.allowed_exts += ['ico', 'svg', 'xhtml', 'xml'] self.allowed_files = [] - self.disallowed_dirs = [ 'CVS' ] + self.disallowed_dirs = ['CVS'] self.recurse = False self.verbose = False self.doc_prefix = "" @@ -127,7 +157,7 @@ def print_help(): def parse_args(): options = OptionsClass() args = [] - + x = 1 while x < len(sys.argv): arg = sys.argv[x] @@ -146,6 +176,8 @@ def parse_args(): sys.exit(0) elif arg == "-p": options.doc_prefix = sys.argv[x] + if options.doc_prefix: + options.doc_prefix = normalize_path(options.doc_prefix) else: values = sys.argv[x].split(",") if arg == "-A": @@ -159,7 +191,7 @@ def parse_args(): else: args.append(sys.argv[x]) x += 1 - + return (options, args) def main(): @@ -168,20 +200,29 @@ def main(): if options.verbose: print("Allowed extensions:", options.allowed_exts) - print("Document prefix : '" + options.doc_prefix + "'") + print("Document prefix : '" + options.doc_prefix + "'") print("Allowed files :", options.allowed_files) success = False - + endswith_slash = (os.sep, os.sep + ".") + for x in args: + trailing_slash = x.endswith(endswith_slash) + x = normalize_path(x) + if trailing_slash: + # Modify behavior of basename and dirname + # as noted in bug #425214, causing foo/ to + # behave similarly to the way that foo/* + # behaves. + x += os.sep basename = os.path.basename(x) dirname = os.path.dirname(x) success |= install(basename, dirname, options) - global skipped_directories for x in skipped_directories: - eqawarn(["QA Notice: dohtml on directory " + \ - "'%s' without recursion option" % x]) + eqawarn(["QA Notice: dohtml on directory '%s' without recursion option" % x]) + for x in skipped_files: + eqawarn(["dohtml: skipped file '%s'" % x]) if success: retcode = 0 diff --git a/portage_with_autodep/bin/ebuild b/portage_with_autodep/bin/ebuild index 35cdc14..262dab6 100755 --- a/portage_with_autodep/bin/ebuild +++ b/portage_with_autodep/bin/ebuild @@ -1,15 +1,16 @@ #!/usr/bin/python -O -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import platform import signal import sys # This block ensures that ^C interrupts are handled quietly. try: - def exithandler(signum,frame): + def exithandler(signum, _frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) sys.exit(128 + signum) @@ -23,56 +24,61 @@ try: except KeyboardInterrupt: sys.exit(128 + signal.SIGINT) -def debug_signal(signum, frame): +def debug_signal(_signum, _frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) -import imp +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) + import io -import optparse import os +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True +from portage import os +from portage import _encodings +from portage import _shell_quote +from portage import _unicode_decode +from portage import _unicode_encode +from portage.const import VDB_PATH +from portage.util._argparse import ArgumentParser +from _emerge.Package import Package +from _emerge.RootConfig import RootConfig description = "See the ebuild(1) man page for more info" usage = "Usage: ebuild <ebuild file> <command> [command] ..." -parser = optparse.OptionParser(description=description, usage=usage) +parser = ArgumentParser(description=description, usage=usage) force_help = "When used together with the digest or manifest " + \ "command, this option forces regeneration of digests for all " + \ "distfiles associated with the current ebuild. Any distfiles " + \ "that do not already exist in ${DISTDIR} will be automatically fetched." -parser.add_option("--force", help=force_help, action="store_true", dest="force") -parser.add_option("--color", help="enable or disable color output", - type="choice", choices=("y", "n")) -parser.add_option("--debug", help="show debug output", - action="store_true", dest="debug") -parser.add_option("--version", help="show version and exit", - action="store_true", dest="version") -parser.add_option("--ignore-default-opts", +parser.add_argument("--force", help=force_help, action="store_true") +parser.add_argument("--color", help="enable or disable color output", + choices=("y", "n")) +parser.add_argument("--debug", help="show debug output", + action="store_true") +parser.add_argument("--version", help="show version and exit", + action="store_true") +parser.add_argument("--ignore-default-opts", action="store_true", help="do not use the EBUILD_DEFAULT_OPTS environment variable") -parser.add_option("--skip-manifest", help="skip all manifest checks", - action="store_true", dest="skip_manifest") - -opts, pargs = parser.parse_args(args=sys.argv[1:]) +parser.add_argument("--skip-manifest", help="skip all manifest checks", + action="store_true") -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage +opts, pargs = parser.parse_known_args(args=sys.argv[1:]) -portage.dep._internal_warnings = True -from portage import os -from portage import _encodings -from portage import _shell_quote -from portage import _unicode_decode -from portage import _unicode_encode -from portage.const import VDB_PATH -from _emerge.Package import Package -from _emerge.RootConfig import RootConfig +def err(txt): + portage.writemsg('ebuild: %s\n' % (txt,), noiselevel=-1) + sys.exit(1) if opts.version: print("Portage", portage.VERSION) @@ -82,8 +88,9 @@ if len(pargs) < 2: parser.error("missing required args") if not opts.ignore_default_opts: - default_opts = portage.settings.get("EBUILD_DEFAULT_OPTS", "").split() - opts, pargs = parser.parse_args(default_opts + sys.argv[1:]) + default_opts = portage.util.shlex_split( + portage.settings.get("EBUILD_DEFAULT_OPTS", "")) + opts, pargs = parser.parse_known_args(default_opts + sys.argv[1:]) debug = opts.debug force = opts.force @@ -112,9 +119,7 @@ if ebuild.endswith(".ebuild"): pf = os.path.basename(ebuild)[:-7] if pf is None: - portage.writemsg("'%s' does not end with '.ebuild'.\n" % \ - (ebuild,), noiselevel=-1) - sys.exit(1) + err("%s: does not end with '.ebuild'" % (ebuild,)) if not os.path.isabs(ebuild): mycwd = os.getcwd() @@ -153,15 +158,14 @@ if ebuild_portdir != vdb_path and \ encoding=_encodings['content'], errors='strict') print("Appending %s to PORTDIR_OVERLAY..." % ebuild_portdir) - imp.reload(portage) + portage._reset_legacy_globals() myrepo = None if ebuild_portdir != vdb_path: myrepo = portage.portdb.getRepositoryName(ebuild_portdir) if not os.path.exists(ebuild): - print("'%s' does not exist." % ebuild) - sys.exit(1) + err('%s: does not exist' % (ebuild,)) ebuild_split = ebuild.split("/") cpv = "%s/%s" % (ebuild_split[-3], pf) @@ -172,8 +176,7 @@ with io.open(_unicode_encode(ebuild, encoding=_encodings['fs'], errors='strict') if eapi is None: eapi = "0" if not portage.catpkgsplit(cpv, eapi=eapi): - print("!!! %s does not follow correct package syntax." % (cpv)) - sys.exit(1) + err('%s: %s: does not follow correct package syntax' % (ebuild, cpv)) if ebuild.startswith(vdb_path): mytree = "vartree" @@ -182,8 +185,7 @@ if ebuild.startswith(vdb_path): portage_ebuild = portage.db[portage.root][mytree].dbapi.findname(cpv, myrepo=myrepo) if os.path.realpath(portage_ebuild) != ebuild: - print("!!! Portage seems to think that %s is at %s" % (cpv, portage_ebuild)) - sys.exit(1) + err('Portage seems to think that %s is at %s' % (cpv, portage_ebuild)) else: mytree = "porttree" @@ -192,12 +194,10 @@ else: portage_ebuild = portage.portdb.findname(cpv, myrepo=myrepo) if not portage_ebuild or portage_ebuild != ebuild: - print("!!! %s does not seem to have a valid PORTDIR structure." % ebuild) - sys.exit(1) + err('%s: does not seem to have a valid PORTDIR structure' % (ebuild,)) if len(pargs) > 1 and "config" in pargs: - print("config must be called on it's own, not combined with any other phase") - sys.exit(1) + err('"config" must not be called with any other phase') def discard_digests(myebuild, mysettings, mydbapi): """Discard all distfiles digests for the given ebuild. This is useful when @@ -306,14 +306,16 @@ def stale_env_warning(): if ebuild_changed: open(os.path.join(tmpsettings['PORTAGE_BUILDDIR'], - '.ebuild_changed'), 'w') + '.ebuild_changed'), 'w').close() from portage.exception import PermissionDenied, \ PortagePackageException, UnsupportedAPIException -if 'digest' in tmpsettings.features and \ - not set(["digest", "manifest"]).intersection(pargs): - pargs = ['digest'] + pargs +if 'digest' in tmpsettings.features: + if pargs and pargs[0] not in ("digest", "manifest"): + pargs = ['digest'] + pargs + # We only need to build digests on the first pass. + tmpsettings.features.discard('digest') checked_for_stale_env = False @@ -327,7 +329,7 @@ for arg in pargs: if arg in ("digest", "manifest") and force: discard_digests(ebuild, tmpsettings, portage.portdb) - a = portage.doebuild(ebuild, arg, portage.root, tmpsettings, + a = portage.doebuild(ebuild, arg, settings=tmpsettings, debug=debug, tree=mytree, vartree=portage.db[portage.root]['vartree']) except KeyboardInterrupt: diff --git a/portage_with_autodep/bin/ebuild-helpers/4/dodoc b/portage_with_autodep/bin/ebuild-helpers/4/dodoc deleted file mode 120000 index 35080ad..0000000 --- a/portage_with_autodep/bin/ebuild-helpers/4/dodoc +++ /dev/null @@ -1 +0,0 @@ -../doins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/4/dohard b/portage_with_autodep/bin/ebuild-helpers/4/dohard deleted file mode 120000 index 1a6b57a..0000000 --- a/portage_with_autodep/bin/ebuild-helpers/4/dohard +++ /dev/null @@ -1 +0,0 @@ -../../banned-helper
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/4/dosed b/portage_with_autodep/bin/ebuild-helpers/4/dosed deleted file mode 120000 index 1a6b57a..0000000 --- a/portage_with_autodep/bin/ebuild-helpers/4/dosed +++ /dev/null @@ -1 +0,0 @@ -../../banned-helper
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/4/prepalldocs b/portage_with_autodep/bin/ebuild-helpers/4/prepalldocs deleted file mode 120000 index 1a6b57a..0000000 --- a/portage_with_autodep/bin/ebuild-helpers/4/prepalldocs +++ /dev/null @@ -1 +0,0 @@ -../../banned-helper
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/dobin b/portage_with_autodep/bin/ebuild-helpers/dobin index f90d893..0ba1eb0 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dobin +++ b/portage_with_autodep/bin/ebuild-helpers/dobin @@ -1,19 +1,20 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ $# -lt 1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ ! -d ${ED}${DESTTREE}/bin ]] ; then - install -d "${ED}${DESTTREE}/bin" || { helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/bin"; exit 2; } + install -d "${ED}${DESTTREE}/bin" || { __helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/bin"; exit 2; } fi ret=0 @@ -28,5 +29,5 @@ for x in "$@" ; do ((ret|=$?)) done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/doconfd b/portage_with_autodep/bin/ebuild-helpers/doconfd index e146000..a3c09a5 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doconfd +++ b/portage_with_autodep/bin/ebuild-helpers/doconfd @@ -4,7 +4,7 @@ if [[ $# -lt 1 ]] ; then source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi diff --git a/portage_with_autodep/bin/ebuild-helpers/dodir b/portage_with_autodep/bin/ebuild-helpers/dodir index 90a3efe..e03ba9a 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dodir +++ b/portage_with_autodep/bin/ebuild-helpers/dodir @@ -1,13 +1,14 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi install -d ${DIROPTIONS} "${@/#/${ED}/}" ret=$? -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/dodoc b/portage_with_autodep/bin/ebuild-helpers/dodoc index 1f333a6..99122c4 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dodoc +++ b/portage_with_autodep/bin/ebuild-helpers/dodoc @@ -1,16 +1,24 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +if ___eapi_dodoc_supports_-r; then + exec \ + env \ + __PORTAGE_HELPER="dodoc" \ + doins "$@" +fi + if [ $# -lt 1 ] ; then - helpers_die "${0##*/}: at least one argument needed" - exit 1 + __helpers_die "${0##*/}: at least one argument needed" + exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi dir="${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" if [ ! -d "${dir}" ] ; then @@ -30,5 +38,5 @@ for x in "$@" ; do fi done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/doenvd b/portage_with_autodep/bin/ebuild-helpers/doenvd index 28ab5d2..9287933 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doenvd +++ b/portage_with_autodep/bin/ebuild-helpers/doenvd @@ -4,7 +4,7 @@ if [[ $# -lt 1 ]] ; then source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi diff --git a/portage_with_autodep/bin/ebuild-helpers/doexe b/portage_with_autodep/bin/ebuild-helpers/doexe index fb228f9..c34fcae 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doexe +++ b/portage_with_autodep/bin/ebuild-helpers/doexe @@ -1,23 +1,23 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ $# -lt 1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ ! -d ${ED}${_E_EXEDESTTREE_} ]] ; then install -d "${ED}${_E_EXEDESTTREE_}" fi -TMP=$T/.doexe_tmp -mkdir "$TMP" +TMP=$(mktemp -d "${T}/.doexe_tmp_XXXXXX") ret=0 @@ -26,7 +26,7 @@ for x in "$@" ; do cp "$x" "$TMP" mysrc=$TMP/${x##*/} elif [ -d "${x}" ] ; then - vecho "doexe: warning, skipping directory ${x}" + __vecho "doexe: warning, skipping directory ${x}" continue else mysrc="${x}" @@ -42,5 +42,5 @@ done rm -rf "$TMP" -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/dohard b/portage_with_autodep/bin/ebuild-helpers/dohard index b52fd7c..e0a44fa 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dohard +++ b/portage_with_autodep/bin/ebuild-helpers/dohard @@ -1,14 +1,22 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +if ! ___eapi_has_dohard; then + die "'${0##*/}' has been banned for EAPI '$EAPI'" + exit 1 +fi + if [[ $# -ne 2 ]] ; then echo "$0: two arguments needed" 1>&2 exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi destdir=${2%/*} [[ ! -d ${ED}${destdir} ]] && dodir "${destdir}" diff --git a/portage_with_autodep/bin/ebuild-helpers/dohtml b/portage_with_autodep/bin/ebuild-helpers/dohtml index 630629a..75d3d00 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dohtml +++ b/portage_with_autodep/bin/ebuild-helpers/dohtml @@ -1,14 +1,19 @@ #!/bin/bash -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh PORTAGE_BIN_PATH=${PORTAGE_BIN_PATH:-/usr/lib/portage/bin} PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} -PYTHONPATH=$PORTAGE_PYM_PATH${PYTHONPATH:+:}$PYTHONPATH \ +# Use safe cwd, avoiding unsafe import for bug #469338. +export __PORTAGE_HELPER_CWD=${PWD} +cd "${PORTAGE_PYM_PATH}" +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/dohtml.py" "$@" ret=$? -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +# Restore cwd for display by __helpers_die +cd "${__PORTAGE_HELPER_CWD}" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/doinfo b/portage_with_autodep/bin/ebuild-helpers/doinfo index 8fd7d45..2edbdc5 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doinfo +++ b/portage_with_autodep/bin/ebuild-helpers/doinfo @@ -1,19 +1,20 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ -z $1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" - exit 1 + __helpers_die "${0##*/}: at least one argument needed" + exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ ! -d ${ED}usr/share/info ]] ; then - install -d "${ED}usr/share/info" || { helpers_die "${0##*/}: failed to install ${ED}usr/share/info"; exit 1; } + install -d "${ED}usr/share/info" || { __helpers_die "${0##*/}: failed to install ${ED}usr/share/info"; exit 1; } fi install -m0644 "$@" "${ED}usr/share/info" @@ -22,6 +23,6 @@ if [ $rval -ne 0 ] ; then for x in "$@" ; do [ -e "$x" ] || echo "!!! ${0##*/}: $x does not exist" 1>&2 done - helpers_die "${0##*/} failed" + __helpers_die "${0##*/} failed" fi exit $rval diff --git a/portage_with_autodep/bin/ebuild-helpers/doinitd b/portage_with_autodep/bin/ebuild-helpers/doinitd index b711e19..476b858 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doinitd +++ b/portage_with_autodep/bin/ebuild-helpers/doinitd @@ -4,7 +4,7 @@ if [[ $# -lt 1 ]] ; then source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi diff --git a/portage_with_autodep/bin/ebuild-helpers/doins b/portage_with_autodep/bin/ebuild-helpers/doins index 443bfdb..c60e057 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doins +++ b/portage_with_autodep/bin/ebuild-helpers/doins @@ -1,14 +1,17 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -if [[ ${0##*/} == dodoc ]] ; then +helper=${__PORTAGE_HELPER:-${0##*/}} + +if [[ ${helper} == dodoc ]] ; then if [ $# -eq 0 ] ; then # default_src_install may call dodoc with no arguments # when DOC is defined but empty, so simply return # sucessfully in this case. + eqawarn "QA Notice: dodoc called with no arguments" exit 0 fi export INSOPTIONS=-m0644 @@ -16,7 +19,7 @@ if [[ ${0##*/} == dodoc ]] ; then fi if [ $# -lt 1 ] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${helper}: at least one argument needed" exit 1 fi @@ -27,28 +30,26 @@ else DOINSRECUR=n fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) export ED="${D}" ;; esac +if ! ___eapi_has_prefix_variables; then + export ED="${D}" +fi if [[ ${INSDESTTREE#${ED}} != "${INSDESTTREE}" ]]; then - vecho "-------------------------------------------------------" 1>&2 - vecho "You should not use \${D} or \${ED} with helpers." 1>&2 - vecho " --> ${INSDESTTREE}" 1>&2 - vecho "-------------------------------------------------------" 1>&2 - helpers_die "${0##*/} used with \${D} or \${ED}" + __vecho "-------------------------------------------------------" 1>&2 + __vecho "You should not use \${D} or \${ED} with helpers." 1>&2 + __vecho " --> ${INSDESTTREE}" 1>&2 + __vecho "-------------------------------------------------------" 1>&2 + __helpers_die "${helper} used with \${D} or \${ED}" exit 1 fi -case "$EAPI" in - 0|1|2|3|3_pre2) - PRESERVE_SYMLINKS=n - ;; - *) - PRESERVE_SYMLINKS=y - ;; -esac +if ___eapi_doins_and_newins_preserve_symlinks; then + PRESERVE_SYMLINKS=y +else + PRESERVE_SYMLINKS=n +fi -export TMP=$T/.doins_tmp +export TMP=$(mktemp -d "${T}/.doins_tmp_XXXXXX") # Use separate directories to avoid potential name collisions. mkdir -p "$TMP"/{1,2} @@ -79,7 +80,7 @@ _doins() { install ${INSOPTIONS} "${mysrc}" "${ED}${INSDESTTREE}/${mydir}" rval=$? [[ -n ${cleanup} ]] && rm -f "${cleanup}" - [ $rval -ne 0 ] && echo "!!! ${0##*/}: $mysrc does not exist" 1>&2 + [ $rval -ne 0 ] && echo "!!! ${helper}: $mysrc does not exist" 1>&2 return $rval } @@ -99,8 +100,8 @@ for x in "$@" ; do if [[ $PRESERVE_SYMLINKS = n && -d $x ]] || \ [[ $PRESERVE_SYMLINKS = y && -d $x && ! -L $x ]] ; then if [ "${DOINSRECUR}" == "n" ] ; then - if [[ ${0##*/} == dodoc ]] ; then - echo "!!! ${0##*/}: $x is a directory" 1>&2 + if [[ ${helper} == dodoc ]] ; then + echo "!!! ${helper}: $x is a directory" 1>&2 ((failed|=1)) fi continue @@ -155,4 +156,4 @@ for x in "$@" ; do fi done rm -rf "$TMP" -[[ $failed -ne 0 || $success -eq 0 ]] && { helpers_die "${0##*/} failed"; exit 1; } || exit 0 +[[ $failed -ne 0 || $success -eq 0 ]] && { __helpers_die "${helper} failed"; exit 1; } || exit 0 diff --git a/portage_with_autodep/bin/ebuild-helpers/dolib b/portage_with_autodep/bin/ebuild-helpers/dolib index 9af5418..fd92d7f 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dolib +++ b/portage_with_autodep/bin/ebuild-helpers/dolib @@ -1,11 +1,12 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi # Setup ABI cruft LIBDIR_VAR="LIBDIR_${ABI}" @@ -19,11 +20,11 @@ libdir="${ED}${DESTTREE}/${CONF_LIBDIR}" if [[ $# -lt 1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi if [[ ! -d ${libdir} ]] ; then - install -d "${libdir}" || { helpers_die "${0##*/}: failed to install ${libdir}"; exit 1; } + install -d "${libdir}" || { __helpers_die "${0##*/}: failed to install ${libdir}"; exit 1; } fi ret=0 @@ -42,5 +43,5 @@ for x in "$@" ; do ((ret|=$?)) done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/doman b/portage_with_autodep/bin/ebuild-helpers/doman index b4047ce..d680859 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doman +++ b/portage_with_autodep/bin/ebuild-helpers/doman @@ -1,16 +1,17 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ $# -lt 1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi i18n="" @@ -58,10 +59,10 @@ for x in "$@" ; do ((ret|=1)) fi else - vecho "doman: '${x}' is probably not a man page; skipping" 1>&2 + __vecho "doman: '${x}' is probably not a man page; skipping" 1>&2 ((ret|=1)) fi done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/domo b/portage_with_autodep/bin/ebuild-helpers/domo index d994343..9a8dda3 100755 --- a/portage_with_autodep/bin/ebuild-helpers/domo +++ b/portage_with_autodep/bin/ebuild-helpers/domo @@ -1,17 +1,18 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh mynum=${#} if [ ${mynum} -lt 1 ] ; then - helpers_die "${0}: at least one argument needed" + __helpers_die "${0}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [ ! -d "${ED}${DESTTREE}/share/locale" ] ; then install -d "${ED}${DESTTREE}/share/locale/" @@ -34,5 +35,5 @@ for x in "$@" ; do ((ret|=$?)) done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/dosbin b/portage_with_autodep/bin/ebuild-helpers/dosbin index d101c8a..361ca83 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosbin +++ b/portage_with_autodep/bin/ebuild-helpers/dosbin @@ -1,19 +1,20 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ $# -lt 1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ ! -d ${ED}${DESTTREE}/sbin ]] ; then - install -d "${ED}${DESTTREE}/sbin" || { helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/sbin"; exit 2; } + install -d "${ED}${DESTTREE}/sbin" || { __helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/sbin"; exit 2; } fi ret=0 @@ -28,5 +29,5 @@ for x in "$@" ; do ((ret|=$?)) done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/dosed b/portage_with_autodep/bin/ebuild-helpers/dosed index f202df7..7db0629 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosed +++ b/portage_with_autodep/bin/ebuild-helpers/dosed @@ -1,14 +1,22 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +if ! ___eapi_has_dosed; then + die "'${0##*/}' has been banned for EAPI '$EAPI'" + exit 1 +fi + if [[ $# -lt 1 ]] ; then echo "!!! ${0##*/}: at least one argument needed" >&2 exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi ret=0 file_found=0 diff --git a/portage_with_autodep/bin/ebuild-helpers/dosym b/portage_with_autodep/bin/ebuild-helpers/dosym index 2489e22..649b100 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosym +++ b/portage_with_autodep/bin/ebuild-helpers/dosym @@ -5,12 +5,13 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ $# -ne 2 ]] ; then - helpers_die "${0##*/}: two arguments needed" + __helpers_die "${0##*/}: two arguments needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ ${2} == */ ]] || \ [[ -d ${ED}${2} && ! -L ${ED}${2} ]] ; then @@ -26,5 +27,5 @@ target="${1}" ln -snf "${target}" "${ED}${2}" ret=$? -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/ecompress b/portage_with_autodep/bin/ebuild-helpers/ecompress index b61421b..71287b4 100755 --- a/portage_with_autodep/bin/ebuild-helpers/ecompress +++ b/portage_with_autodep/bin/ebuild-helpers/ecompress @@ -5,7 +5,7 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh if [[ -z $1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi @@ -68,7 +68,7 @@ decompress_args() { case $1 in --suffix) - [[ -n $2 ]] && vecho "${0##*/}: --suffix takes no additional arguments" 1>&2 + [[ -n $2 ]] && __vecho "${0##*/}: --suffix takes no additional arguments" 1>&2 if [[ ! -e ${T}/.ecompress.suffix ]] ; then set -e @@ -93,7 +93,7 @@ case $1 in cat "${T}/.ecompress.suffix" ;; --bin) - [[ -n $2 ]] && vecho "${0##*/}: --bin takes no additional arguments" 1>&2 + [[ -n $2 ]] && __vecho "${0##*/}: --bin takes no additional arguments" 1>&2 echo "${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}" ;; @@ -104,18 +104,18 @@ case $1 in >> "$x" ((ret|=$?)) done - [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" + [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret ;; --dequeue) - [[ -n $2 ]] && vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 + [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 find "${D}" -name '*.ecompress.file' -print0 \ | sed -e 's:\.ecompress\.file::g' \ | ${XARGS} -0 ecompress find "${D}" -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f ;; --*) - helpers_die "${0##*/}: unknown arguments '$*'" + __helpers_die "${0##*/}: unknown arguments '$*'" exit 1 ;; *) @@ -155,7 +155,7 @@ case $1 in # Finally, let's actually do some real work "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@" ret=$? - [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" + [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret ;; esac diff --git a/portage_with_autodep/bin/ebuild-helpers/ecompressdir b/portage_with_autodep/bin/ebuild-helpers/ecompressdir index a2c9e52..eca5888 100755 --- a/portage_with_autodep/bin/ebuild-helpers/ecompressdir +++ b/portage_with_autodep/bin/ebuild-helpers/ecompressdir @@ -1,18 +1,21 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh if [[ -z $1 ]] ; then - helpers_die "${0##*/}: at least one argument needed" + __helpers_die "${0##*/}: at least one argument needed" exit 1 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} EPREFIX= ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} EPREFIX= +fi -case $1 in +SIZE_LIMIT='' +while [[ $# -gt 0 ]] ; do + case $1 in --ignore) shift for skip in "$@" ; do @@ -27,45 +30,66 @@ case $1 in set -- "${@/#/${ED}}" ret=0 for x in "$@" ; do - >> "$x" + # Stash the limit in the .dir file so we can reload it later. + printf "${SIZE_LIMIT}" > "${x}" ((ret|=$?)) done - [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" + [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret ;; --dequeue) - [[ -n $2 ]] && vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 + [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 find "${ED}" -name '*.ecompress.dir' -print0 \ | sed -e 's:\.ecompress\.dir::g' -e "s:${ED}:/:g" \ | ${XARGS} -0 ecompressdir find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f exit 0 ;; + --limit) + SIZE_LIMIT=$2 + shift + ;; --*) - helpers_die "${0##*/}: unknown arguments '$*'" + __helpers_die "${0##*/}: unknown arguments '$*'" exit 1 ;; -esac + *) + break + ;; + esac + shift +done # figure out the new suffix suffix=$(ecompress --suffix) -# funk_up_dir(action, suffix, binary) +# funk_up_dir(action, suffix, binary, [size_limit]) # - action: compress or decompress # - suffix: the compression suffix to work with # - binary: the program to execute that'll compress/decompress +# - size_limit: if compressing, skip files smaller than this # The directory we act on is implied in the ${dir} variable funk_up_dir() { - local act=$1 suffix=$2 binary=$3 + local act=$1 suffix=$2 binary=$3 size_limit=$4 local negate="" [[ ${act} == "compress" ]] && negate="!" + local ret=0 # first we act on all the files - find "${dir}" -type f ${negate} -iname '*'${suffix} -print0 | ${XARGS} -0 ${binary} + local args=( + -type f + ${negate} -iname "*${suffix}" + ) + [[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" ) + find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary} ((ret|=$?)) - find "${dir}" -type l -print0 | \ + # Repeat until nothing changes, in order to handle multiple + # levels of indirection (see bug #470916). + local -i indirection=0 + while true ; do + local something_changed= while read -r -d $'\0' brokenlink ; do [[ -e ${brokenlink} ]] && continue olddest=$(readlink "${brokenlink}") @@ -80,15 +104,34 @@ funk_up_dir() { skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED}}/${brokenlink%/*}/${olddest} fi [[ -e ${skip_dir_dest} ]] && continue - [[ ${act} == "compress" ]] \ - && newdest="${olddest}${suffix}" \ - || newdest="${olddest%${suffix}}" + if [[ ${act} == "compress" ]] ; then + newdest=${olddest}${suffix} + else + [[ ${olddest} == *${suffix} ]] || continue + newdest=${olddest%${suffix}} + fi + if [[ "${newdest}" == /* ]] ; then + [[ -f "${D}${newdest}" ]] || continue + else + [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue + fi + something_changed=${brokenlink} rm -f "${brokenlink}" [[ ${act} == "compress" ]] \ && ln -snf "${newdest}" "${brokenlink}${suffix}" \ || ln -snf "${newdest}" "${brokenlink%${suffix}}" ((ret|=$?)) + done < <(find "${dir}" -type l -print0) + [[ -n ${something_changed} ]] || break + (( indirection++ )) + if (( indirection >= 100 )) ; then + # Protect against possibility of a bug triggering an endless loop. + eerror "ecompressdir: too many levels of indirection for" \ + "'${actual_dir#${ED}}/${something_changed#./}'" + break + fi done + return ${ret} } # _relocate_skip_dirs(srctree, dsttree) @@ -124,13 +167,13 @@ decompressors=( ".lzma" "unxz -f" ) -multijob_init +__multijob_init for dir in "$@" ; do dir=${dir#/} dir="${ED}${dir}" if [[ ! -d ${dir} ]] ; then - vecho "${0##*/}: /${dir#${ED}} does not exist!" + __vecho "${0##*/}: /${dir#${ED}} does not exist!" continue fi cd "${dir}" @@ -142,39 +185,41 @@ for dir in "$@" ; do # since we've been requested to compress the whole dir, # delete any individual queued requests + size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")} rm -f "${actual_dir}.ecompress.dir" find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f # not uncommon for packages to compress doc files themselves - for (( d = 0; d < ${#decompressors[@]}; d += 2 )) ; do + for (( i = 0; i < ${#decompressors[@]}; i += 2 )) ; do # It's faster to parallelize at this stage than to try to # parallelize the compressors. This is because the find|xargs # ends up launching less compressors overall, so the overhead # of forking children ends up dominating. ( - multijob_child_init + __multijob_child_init funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}" ) & - multijob_post_fork + __multijob_post_fork : $(( ret |= $? )) done + __multijob_finish + : $(( ret |= $? )) + # forcibly break all hard links as some compressors whine about it find "${dir}" -type f -links +1 -exec env file="{}" sh -c \ 'cp -p "${file}" "${file}.ecompress.break" ; mv -f "${file}.ecompress.break" "${file}"' \; - multijob_finish - : $(( ret |= $? )) - # now lets do our work if [[ -n ${suffix} ]] ; then - vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}" - funk_up_dir "compress" "${suffix}" "ecompress" + __vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}" + funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}" + : $(( ret |= $? )) fi # finally, restore the skipped stuff restore_skip_dirs done -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/emake b/portage_with_autodep/bin/ebuild-helpers/emake index d842781..69d836f 100755 --- a/portage_with_autodep/bin/ebuild-helpers/emake +++ b/portage_with_autodep/bin/ebuild-helpers/emake @@ -24,5 +24,5 @@ fi ${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@" ret=$? -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/fowners b/portage_with_autodep/bin/ebuild-helpers/fowners index a213c9e..cee4108 100755 --- a/portage_with_autodep/bin/ebuild-helpers/fowners +++ b/portage_with_autodep/bin/ebuild-helpers/fowners @@ -4,8 +4,9 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) EPREFIX= ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + EPREFIX= ED=${D} +fi # we can't prefix all arguments because # chown takes random options @@ -13,10 +14,5 @@ slash="/" chown "${@/#${slash}/${ED}${slash}}" ret=$? -if [[ ${ret} != 0 && -n ${EPREFIX} && ${EUID} != 0 ]] ; then - ewarn "fowners failure ignored in Prefix with non-privileged user" - exit 0 -fi - -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/fperms b/portage_with_autodep/bin/ebuild-helpers/fperms index a2f77ea..d854ebb 100755 --- a/portage_with_autodep/bin/ebuild-helpers/fperms +++ b/portage_with_autodep/bin/ebuild-helpers/fperms @@ -1,16 +1,17 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi # we can't prefix all arguments because # chmod takes random options slash="/" chmod "${@/#${slash}/${ED}${slash}}" ret=$? -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/newbin b/portage_with_autodep/bin/ebuild-helpers/newbin index bf98744..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newbin +++ b/portage_with_autodep/bin/ebuild-helpers/newbin @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec dobin "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newconfd b/portage_with_autodep/bin/ebuild-helpers/newconfd index fa3710d..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newconfd +++ b/portage_with_autodep/bin/ebuild-helpers/newconfd @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec doconfd "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newdoc b/portage_with_autodep/bin/ebuild-helpers/newdoc index df6fb1d..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newdoc +++ b/portage_with_autodep/bin/ebuild-helpers/newdoc @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec dodoc "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newenvd b/portage_with_autodep/bin/ebuild-helpers/newenvd index c54af05..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newenvd +++ b/portage_with_autodep/bin/ebuild-helpers/newenvd @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec doenvd "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newexe b/portage_with_autodep/bin/ebuild-helpers/newexe index 9bcf64b..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newexe +++ b/portage_with_autodep/bin/ebuild-helpers/newexe @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec doexe "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newinitd b/portage_with_autodep/bin/ebuild-helpers/newinitd index 03bbe68..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newinitd +++ b/portage_with_autodep/bin/ebuild-helpers/newinitd @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec doinitd "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newins b/portage_with_autodep/bin/ebuild-helpers/newins index adf2d80..0335985 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newins +++ b/portage_with_autodep/bin/ebuild-helpers/newins @@ -1,38 +1,57 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +helper=${0##*/} + if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" + __helpers_die "${helper}: Need two arguments, old file and new file" exit 1 fi -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 +(($#>2)) && \ + eqawarn "QA Notice: ${helper} called with more than 2 arguments: ${@:3}" + +stdin= +if ___eapi_newins_supports_reading_from_standard_input && [[ $1 == "-" ]]; then + stdin=yes fi -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" || exit $? -case "$EAPI" in - 0|1|2|3|3_pre2) - cp "$1" "$T/$2" || exit $? - ;; - *) - cp -P "$1" "$T/$2" - ret=$? - if [[ $ret -ne 0 ]] ; then - helpers_die "${0##*/} failed" - exit $ret +TMP=$(mktemp -d "${T}/.newins_tmp_XXXXXX") +trap 'rm -rf "${TMP}"' EXIT + +if [[ ${stdin} ]] ; then + if [[ -t 0 ]] ; then + __helpers_die "!!! ${helper}: Input is from a terminal" + exit 1 + fi + cat > "${TMP}/$2" + ret=$? +else + if [[ ! -e $1 ]] ; then + __helpers_die "!!! ${helper}: $1 does not exist" + exit 1 + fi + + cp_args="-f" + if [[ ${helper} == newins ]] ; then + if ___eapi_doins_and_newins_preserve_symlinks; then + cp_args+=" -P" fi - ;; -esac -doins "${T}/${2}" + fi + + cp ${cp_args} "$1" "${TMP}/$2" + ret=$? +fi + +if [[ ${ret} -ne 0 ]] ; then + __helpers_die "${0##*/} failed" + exit ${ret} +fi + +do${helper#new} "${TMP}/$2" ret=$? -rm -rf "${T}/${2}" -[[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" +[[ $ret -ne 0 ]] && __helpers_die "${helper} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/newlib.a b/portage_with_autodep/bin/ebuild-helpers/newlib.a index 7ff8195..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newlib.a +++ b/portage_with_autodep/bin/ebuild-helpers/newlib.a @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec dolib.a "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newlib.so b/portage_with_autodep/bin/ebuild-helpers/newlib.so index fd4c097..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newlib.so +++ b/portage_with_autodep/bin/ebuild-helpers/newlib.so @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec dolib.so "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newman b/portage_with_autodep/bin/ebuild-helpers/newman index 889e0f9..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newman +++ b/portage_with_autodep/bin/ebuild-helpers/newman @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec doman "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newsbin b/portage_with_autodep/bin/ebuild-helpers/newsbin index 9df0af2..59a0db2 100755..120000 --- a/portage_with_autodep/bin/ebuild-helpers/newsbin +++ b/portage_with_autodep/bin/ebuild-helpers/newsbin @@ -1,22 +1 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -if [[ -z ${T} ]] || [[ -z ${2} ]] ; then - helpers_die "${0##*/}: Need two arguments, old file and new file" - exit 1 -fi - -if [ ! -e "$1" ] ; then - helpers_die "!!! ${0##*/}: $1 does not exist" - exit 1 -fi - -(($#>2)) && \ - eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" - -rm -rf "${T}/${2}" && \ -cp -f "${1}" "${T}/${2}" && \ -exec dosbin "${T}/${2}" +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/portageq b/portage_with_autodep/bin/ebuild-helpers/portageq index ec30b66..b67b03f 100755 --- a/portage_with_autodep/bin/ebuild-helpers/portageq +++ b/portage_with_autodep/bin/ebuild-helpers/portageq @@ -1,8 +1,10 @@ #!/bin/bash -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 PORTAGE_BIN_PATH=${PORTAGE_BIN_PATH:-/usr/lib/portage/bin} PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} -PYTHONPATH=$PORTAGE_PYM_PATH${PYTHONPATH:+:}$PYTHONPATH \ +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/portageq" "$@" diff --git a/portage_with_autodep/bin/ebuild-helpers/prepall b/portage_with_autodep/bin/ebuild-helpers/prepall index 49e646c..fb5c2db 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepall +++ b/portage_with_autodep/bin/ebuild-helpers/prepall @@ -1,11 +1,12 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if has chflags $FEATURES ; then # Save all the file flags for restoration at the end of prepall. diff --git a/portage_with_autodep/bin/ebuild-helpers/prepalldocs b/portage_with_autodep/bin/ebuild-helpers/prepalldocs index 560a02b..3094661 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepalldocs +++ b/portage_with_autodep/bin/ebuild-helpers/prepalldocs @@ -1,15 +1,21 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +if ___eapi_has_docompress; then + die "'${0##*/}' has been banned for EAPI '$EAPI'" + exit 1 +fi + if [[ -n $1 ]] ; then - vecho "${0##*/}: invalid usage; takes no arguments" 1>&2 + __vecho "${0##*/}: invalid usage; takes no arguments" 1>&2 fi -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi [[ -d ${ED}usr/share/doc ]] || exit 0 diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallinfo b/portage_with_autodep/bin/ebuild-helpers/prepallinfo index db9bbfa..1a20275 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallinfo +++ b/portage_with_autodep/bin/ebuild-helpers/prepallinfo @@ -1,11 +1,12 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi [[ -d ${ED}usr/share/info ]] || exit 0 diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallman b/portage_with_autodep/bin/ebuild-helpers/prepallman index dee1c72..5331eaf 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallman +++ b/portage_with_autodep/bin/ebuild-helpers/prepallman @@ -1,22 +1,22 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh # replaced by controllable compression in EAPI 4 -has "${EAPI}" 0 1 2 3 || exit 0 +___eapi_has_docompress && exit 0 -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi ret=0 -find "${ED}" -type d -name man > "${T}"/prepallman.filelist -while read -r mandir ; do +while IFS= read -r -d '' mandir ; do mandir=${mandir#${ED}} prepman "${mandir%/man}" ((ret|=$?)) -done < "${T}"/prepallman.filelist +done < <(find "${ED}" -type d -name man -print0) exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallstrip b/portage_with_autodep/bin/ebuild-helpers/prepallstrip index 28320d9..1aa6686 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallstrip +++ b/portage_with_autodep/bin/ebuild-helpers/prepallstrip @@ -1,8 +1,11 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi exec prepstrip "${ED}" diff --git a/portage_with_autodep/bin/ebuild-helpers/prepinfo b/portage_with_autodep/bin/ebuild-helpers/prepinfo index ffe2ece..5afc18a 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepinfo +++ b/portage_with_autodep/bin/ebuild-helpers/prepinfo @@ -1,11 +1,12 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi if [[ -z $1 ]] ; then infodir="/usr/share/info" @@ -19,7 +20,7 @@ fi if [[ ! -d ${ED}${infodir} ]] ; then if [[ -n $1 ]] ; then - vecho "${0##*/}: '${infodir}' does not exist!" + __vecho "${0##*/}: '${infodir}' does not exist!" exit 1 else exit 0 @@ -33,5 +34,5 @@ find "${ED}${infodir}" -type d -print0 | while read -r -d $'\0' x ; do rm -f "${x}"/dir{,.info}{,.gz,.bz2} done -has "${EAPI}" 0 1 2 3 || exit 0 +___eapi_has_docompress && exit 0 exec ecompressdir --queue "${infodir}" diff --git a/portage_with_autodep/bin/ebuild-helpers/preplib b/portage_with_autodep/bin/ebuild-helpers/preplib deleted file mode 100755 index 6e91cf3..0000000 --- a/portage_with_autodep/bin/ebuild-helpers/preplib +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 1999-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh - -eqawarn "QA Notice: Deprecated call to 'preplib'" - -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac - -LIBDIR_VAR="LIBDIR_${ABI}" -if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" -fi -unset LIBDIR_VAR - -if [ -z "${CONF_LIBDIR}" ]; then - # we need this to default to lib so that things dont break - CONF_LIBDIR="lib" -fi - -if [ -z "$1" ] ; then - z="${ED}usr/${CONF_LIBDIR}" -else - z="${ED}$1/${CONF_LIBDIR}" -fi - -if [ -d "${z}" ] ; then - ldconfig -n -N "${z}" -fi diff --git a/portage_with_autodep/bin/ebuild-helpers/prepman b/portage_with_autodep/bin/ebuild-helpers/prepman index f96b641..fb5dcb4 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepman +++ b/portage_with_autodep/bin/ebuild-helpers/prepman @@ -1,13 +1,17 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +# Do not compress man pages which are smaller than this (in bytes). #169260 +SIZE_LIMIT='128' + source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi -if [[ -z $1 ]] ; then +if [[ -z $1 ]] ; then mandir="${ED}usr/share/man" else mandir="${ED}$1/man" @@ -19,7 +23,7 @@ if [[ ! -d ${mandir} ]] ; then fi # replaced by controllable compression in EAPI 4 -has "${EAPI}" 0 1 2 3 || exit 0 +___eapi_has_docompress && exit 0 shopt -s nullglob @@ -30,6 +34,6 @@ for subdir in "${mandir}"/man* "${mandir}"/*/man* ; do [[ -d ${subdir} ]] && really_is_mandir=1 && break done -[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --queue "${mandir#${ED}}" +[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --limit ${SIZE_LIMIT} --queue "${mandir#${ED}}" exit 0 diff --git a/portage_with_autodep/bin/ebuild-helpers/prepstrip b/portage_with_autodep/bin/ebuild-helpers/prepstrip index 85d5d6a..2332388 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepstrip +++ b/portage_with_autodep/bin/ebuild-helpers/prepstrip @@ -1,7 +1,8 @@ #!/bin/bash -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh # avoid multiple calls to `has`. this creates things like: @@ -15,11 +16,12 @@ exp_tf() { eval ${var}_${flag}=$(tf has ${flag} ${!var}) done } -exp_tf FEATURES compressdebug installsources nostrip splitdebug -exp_tf RESTRICT binchecks installsources strip +exp_tf FEATURES compressdebug installsources nostrip splitdebug xattr +exp_tf RESTRICT binchecks installsources splitdebug strip -[[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "${EAPI}" in 0|1|2) EPREFIX= ED=${D} ;; esac +if ! ___eapi_has_prefix_variables; then + EPREFIX= ED=${D} +fi banner=false SKIP_STRIP=false @@ -29,6 +31,30 @@ if ${RESTRICT_strip} || ${FEATURES_nostrip} ; then ${FEATURES_installsources} || exit 0 fi +PRESERVE_XATTR=false +if [[ ${KERNEL} == linux ]] && ${FEATURES_xattr} ; then + PRESERVE_XATTR=true + if type -P getfattr >/dev/null && type -P setfattr >/dev/null ; then + dump_xattrs() { + getfattr -d --absolute-names "$1" + } + restore_xattrs() { + setfattr --restore=- + } + else + dump_xattrs() { + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + "${PORTAGE_PYTHON:-/usr/bin/python}" \ + "${PORTAGE_BIN_PATH}/xattr-helper.py" --dump < <(echo -n "$1") + } + restore_xattrs() { + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + "${PORTAGE_PYTHON:-/usr/bin/python}" \ + "${PORTAGE_BIN_PATH}/xattr-helper.py" --restore + } + fi +fi + # look up the tools we might be using for t in STRIP:strip OBJCOPY:objcopy READELF:readelf ; do v=${t%:*} # STRIP @@ -51,7 +77,7 @@ case $(${STRIP} --version 2>/dev/null) in # We'll leave out -R .note for now until we can check out the relevance # of the section when it has the ALLOC flag set on it ... SAFE_STRIP_FLAGS="--strip-unneeded" - DEF_STRIP_FLAGS="-R .comment -R .GCC.command.line" + DEF_STRIP_FLAGS="-R .comment -R .GCC.command.line -R .note.gnu.gold-version" SPLIT_STRIP_FLAGS= ;; esac @@ -62,23 +88,13 @@ prepstrip_sources_dir=${EPREFIX}/usr/src/debug/${CATEGORY}/${PF} type -P debugedit >/dev/null && debugedit_found=true || debugedit_found=false debugedit_warned=false -multijob_init +__multijob_init # Setup $T filesystem layout that we care about. tmpdir="${T}/prepstrip" rm -rf "${tmpdir}" mkdir -p "${tmpdir}"/{inodes,splitdebug,sources} -# Usage: inode_var_name: <file> -inode_file_link() { - echo -n "${tmpdir}/inodes/" - if [[ ${USERLAND} == "BSD" ]] ; then - stat -f '%i' "$1" - else - stat -c '%i' "$1" - fi -} - # Usage: save_elf_sources <elf> save_elf_sources() { ${FEATURES_installsources} || return 0 @@ -93,7 +109,6 @@ save_elf_sources() { fi local x=$1 - [[ -f $(inode_file_link "${x}") ]] && return 0 # since we're editing the ELF here, we should recompute the build-id # (the -i flag below). save that output so we don't need to recompute @@ -101,20 +116,22 @@ save_elf_sources() { buildid=$(debugedit -i \ -b "${WORKDIR}" \ -d "${prepstrip_sources_dir}" \ - -l "${tmpdir}/sources/${x##*/}.${BASHPID}" \ + -l "${tmpdir}/sources/${x##*/}.${BASHPID:-$(__bashpid)}" \ "${x}") } # Usage: save_elf_debug <elf> [splitdebug file] save_elf_debug() { ${FEATURES_splitdebug} || return 0 + ${RESTRICT_splitdebug} && return 0 # NOTE: Debug files must be installed in # ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs # twice in this path) in order for gdb's debug-file-directory # lookup to work correctly. local x=$1 - local splitdebug=$2 + local inode_debug=$2 + local splitdebug=$3 local y=${ED}usr/lib/debug/${x:${#D}}.debug # dont save debug info twice @@ -122,9 +139,8 @@ save_elf_debug() { mkdir -p "${y%/*}" - local inode=$(inode_file_link "${x}") - if [[ -f ${inode} ]] ; then - ln "${inode}" "${y}" + if [ -f "${inode_debug}" ] ; then + ln "${inode_debug}" "${y}" || die "ln failed unexpectedly" else if [[ -n ${splitdebug} ]] ; then mv "${splitdebug}" "${y}" @@ -134,64 +150,89 @@ save_elf_debug() { ${OBJCOPY} ${objcopy_flags} "${x}" "${y}" ${OBJCOPY} --add-gnu-debuglink="${y}" "${x}" fi - local args="a-x,o-w" - [[ -g ${x} || -u ${x} ]] && args+=",go-r" - chmod ${args} "${y}" - ln "${y}" "${inode}" + # Only do the following if the debug file was + # successfully created (see bug #446774). + if [ $? -eq 0 ] ; then + local args="a-x,o-w" + [[ -g ${x} || -u ${x} ]] && args+=",go-r" + chmod ${args} "${y}" + ln "${y}" "${inode_debug}" || die "ln failed unexpectedly" + fi fi # if we don't already have build-id from debugedit, look it up if [[ -z ${buildid} ]] ; then # convert the readelf output to something useful - buildid=$(${READELF} -x .note.gnu.build-id "${x}" 2>/dev/null \ - | awk '$NF ~ /GNU/ { getline; printf $2$3$4$5; getline; print $2 }') + buildid=$(${READELF} -n "${x}" 2>/dev/null | awk '/Build ID:/{ print $NF; exit }') fi if [[ -n ${buildid} ]] ; then local buildid_dir="${ED}usr/lib/debug/.build-id/${buildid:0:2}" local buildid_file="${buildid_dir}/${buildid:2}" mkdir -p "${buildid_dir}" - ln -s "../../${x:${#D}}.debug" "${buildid_file}.debug" - ln -s "/${x:${#D}}" "${buildid_file}" + [ -L "${buildid_file}".debug ] || ln -s "../../${x:${#D}}.debug" "${buildid_file}.debug" + [ -L "${buildid_file}" ] || ln -s "/${x:${#D}}" "${buildid_file}" fi } # Usage: process_elf <elf> process_elf() { - local x=$1 strip_flags=${*:2} - - vecho " ${x:${#ED}}" - save_elf_sources "${x}" + local x=$1 inode_link=$2 strip_flags=${*:3} + local already_stripped lockfile xt_data + + __vecho " ${x:${#ED}}" + + # If two processes try to debugedit or strip the same hardlink at the + # same time, it may corrupt files or cause loss of splitdebug info. + # So, use a lockfile to prevent interference (easily observed with + # dev-vcs/git which creates ~111 hardlinks to one file in + # /usr/libexec/git-core). + lockfile=${inode_link}_lockfile + if ! ln "${inode_link}" "${lockfile}" 2>/dev/null ; then + while [[ -f ${lockfile} ]] ; do + sleep 1 + done + unset lockfile + fi - if ${strip_this} ; then + [ -f "${inode_link}_stripped" ] && already_stripped=true || already_stripped=false - # If two processes try to strip the same hardlink at the same - # time, it will cause one of them to lose the splitdebug info. - # So, use a lockfile to prevent interference (easily observed - # with dev-vcs/git which creates ~109 hardlinks to one file in - # /usr/libexec/git-core). - local lockfile=$(inode_file_link "${x}")_lockfile - if ! ln "${x}" "${lockfile}" ; then - while [[ -f ${lockfile} ]] ; do - sleep 1 - done - unset lockfile + if ! ${already_stripped} ; then + if ${PRESERVE_XATTR} ; then + xt_data=$(dump_xattrs "${x}") fi + save_elf_sources "${x}" + fi + + if ${strip_this} ; then # see if we can split & strip at the same time if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then local shortname="${x##*/}.debug" - local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID}" + local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID:-$(__bashpid)}" + ${already_stripped} || \ ${STRIP} ${strip_flags} \ -f "${splitdebug}" \ -F "${shortname}" \ "${x}" - save_elf_debug "${x}" "${splitdebug}" + save_elf_debug "${x}" "${inode_link}_debug" "${splitdebug}" else - save_elf_debug "${x}" + save_elf_debug "${x}" "${inode_link}_debug" + ${already_stripped} || \ ${STRIP} ${strip_flags} "${x}" fi - [[ -n ${lockfile} ]] && rm -f "${lockfile}" fi + + if ${already_stripped} ; then + rm -f "${x}" || die "rm failed unexpectedly" + ln "${inode_link}_stripped" "${x}" || die "ln failed unexpectedly" + else + ln "${x}" "${inode_link}_stripped" || die "ln failed unexpectedly" + if [[ ${xt_data} ]] ; then + restore_xattrs <<< "${xt_data}" + fi + fi + + [[ -n ${lockfile} ]] && rm -f "${lockfile}" } # The existance of the section .symtab tells us that a binary is stripped. @@ -204,7 +245,7 @@ if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then log=${tmpdir}/scanelf-already-stripped.log scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "${log}" ( - multijob_child_init + __multijob_child_init qa_var="QA_PRESTRIPPED_${ARCH/-/_}" [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}" if [[ -n ${QA_PRESTRIPPED} && -s ${log} && \ @@ -219,28 +260,49 @@ if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then fi sed -e "/^\$/d" -e "s#^#/#" -i "${log}" if [[ -s ${log} ]] ; then - vecho -e "\n" + __vecho -e "\n" eqawarn "QA Notice: Pre-stripped files found:" eqawarn "$(<"${log}")" else rm -f "${log}" fi ) & - multijob_post_fork + __multijob_post_fork +fi + +# Since strip creates a new inode, we need to know the initial set of +# inodes in advance, so that we can avoid interference due to trying +# to strip the same (hardlinked) file multiple times in parallel. +# See bug #421099. +if [[ ${USERLAND} == BSD ]] ; then + get_inode_number() { stat -f '%i' "$1"; } +else + get_inode_number() { stat -c '%i' "$1"; } fi +cd "${tmpdir}/inodes" || die "cd failed unexpectedly" +while read -r x ; do + inode_link=$(get_inode_number "${x}") || die "stat failed unexpectedly" + echo "${x}" >> "${inode_link}" || die "echo failed unexpectedly" +done < <( + # Use sort -u to eliminate duplicates for bug #445336. + ( + scanelf -yqRBF '#k%F' -k '.symtab' "$@" + find "$@" -type f ! -type l -name '*.a' + ) | LC_ALL=C sort -u +) # Now we look for unstripped binaries. -for x in \ - $(scanelf -yqRBF '#k%F' -k '.symtab' "$@") \ - $(find "$@" -type f -name '*.a') +for inode_link in $(shopt -s nullglob; echo *) ; do +while read -r x do + if ! ${banner} ; then - vecho "strip: ${STRIP} ${PORTAGE_STRIP_FLAGS}" + __vecho "strip: ${STRIP} ${PORTAGE_STRIP_FLAGS}" banner=true fi ( - multijob_child_init + __multijob_child_init f=$(file "${x}") || exit 0 [[ -z ${f} ]] && exit 0 @@ -275,27 +337,34 @@ do buildid= if [[ ${f} == *"current ar archive"* ]] ; then - vecho " ${x:${#ED}}" + __vecho " ${x:${#ED}}" if ${strip_this} ; then - # hmm, can we split debug/sources for .a ? - ${STRIP} -g "${x}" + # If we have split debug enabled, then do not strip this. + # There is no concept of splitdebug for objects not yet + # linked in (only for finally linked ELFs), so we have to + # retain the debug info in the archive itself. + if ! ${FEATURES_splitdebug} || ${RESTRICT_splitdebug} ; then + ${STRIP} -g "${x}" + fi fi elif [[ ${f} == *"SB executable"* || ${f} == *"SB shared object"* ]] ; then - process_elf "${x}" ${PORTAGE_STRIP_FLAGS} + process_elf "${x}" "${inode_link}" ${PORTAGE_STRIP_FLAGS} elif [[ ${f} == *"SB relocatable"* ]] ; then - process_elf "${x}" ${SAFE_STRIP_FLAGS} + process_elf "${x}" "${inode_link}" ${SAFE_STRIP_FLAGS} fi if ${was_not_writable} ; then chmod u-w "${x}" fi ) & - multijob_post_fork + __multijob_post_fork + +done < "${inode_link}" done # With a bit more work, we could run the rsync processes below in # parallel, but not sure that'd be an overall improvement. -multijob_finish +__multijob_finish cd "${tmpdir}"/sources/ && cat * > "${tmpdir}/debug.sources" 2>/dev/null if [[ -s ${tmpdir}/debug.sources ]] && \ @@ -303,11 +372,11 @@ if [[ -s ${tmpdir}/debug.sources ]] && \ ! ${RESTRICT_installsources} && \ ${debugedit_found} then - vecho "installsources: rsyncing source files" + __vecho "installsources: rsyncing source files" [[ -d ${D}${prepstrip_sources_dir} ]] || mkdir -p "${D}${prepstrip_sources_dir}" grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \ (cd "${WORKDIR}"; LANG=C sort -z -u | \ - rsync -tL0 --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" ) + rsync -tL0 --chmod=ugo-st,a+r,go-w,Da+x,Fa-x --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" ) # Preserve directory structure. # Needed after running save_elf_sources. @@ -318,4 +387,5 @@ then done < <(find "${D}${prepstrip_sources_dir}/" -type d -empty -print0) fi +cd "${T}" rm -rf "${tmpdir}" diff --git a/portage_with_autodep/bin/ebuild-ipc b/portage_with_autodep/bin/ebuild-ipc index 43e4a02..820005f 100755 --- a/portage_with_autodep/bin/ebuild-ipc +++ b/portage_with_autodep/bin/ebuild-ipc @@ -1,8 +1,10 @@ #!/bin/bash -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 PORTAGE_BIN_PATH=${PORTAGE_BIN_PATH:-/usr/lib/portage/bin} PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} -PYTHONPATH=$PORTAGE_PYM_PATH${PYTHONPATH:+:}$PYTHONPATH \ +# Use safe cwd, avoiding unsafe import for bug #469338. +cd "${PORTAGE_PYM_PATH}" +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ exec "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH/ebuild-ipc.py" "$@" diff --git a/portage_with_autodep/bin/ebuild-ipc.py b/portage_with_autodep/bin/ebuild-ipc.py index 29d4c23..d351e94 100755 --- a/portage_with_autodep/bin/ebuild-ipc.py +++ b/portage_with_autodep/bin/ebuild-ipc.py @@ -1,24 +1,28 @@ #!/usr/bin/python -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # This is a helper which ebuild processes can use # to communicate with portage's main python process. -import errno import logging import os import pickle -import select +import platform import signal import sys import time -import traceback def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Avoid sandbox violations after python upgrade. pym_path = os.path.join(os.path.dirname( @@ -31,14 +35,28 @@ if os.environ.get("SANDBOX_ON") == "1": ":".join(filter(None, sandbox_write)) import portage +portage._internal_caller = True portage._disable_legacy_globals() +from portage.util._async.ForkProcess import ForkProcess +from portage.util._eventloop.global_event_loop import global_event_loop +from _emerge.PipeReader import PipeReader + +class FifoWriter(ForkProcess): + + __slots__ = ('buf', 'fifo',) + + def _run(self): + # Atomically write the whole buffer into the fifo. + with open(self.fifo, 'wb', 0) as f: + f.write(self.buf) + return os.EX_OK + class EbuildIpc(object): # Timeout for each individual communication attempt (we retry # as long as the daemon process appears to be alive). - _COMMUNICATE_RETRY_TIMEOUT_SECONDS = 15 - _BUFSIZE = 4096 + _COMMUNICATE_RETRY_TIMEOUT_MS = 15000 def __init__(self): self.fifo_dir = os.environ['PORTAGE_BUILDDIR'] @@ -82,7 +100,7 @@ class EbuildIpc(object): 'ebuild-ipc: daemon process not detected\n'), level=logging.ERROR, noiselevel=-1) - def _wait(self, pid, pr, msg): + def _run_writer(self, fifo_writer, msg): """ Wait on pid and return an appropriate exit code. This may return unsuccessfully due to timeout if the daemon @@ -91,88 +109,48 @@ class EbuildIpc(object): start_time = time.time() - while True: - try: - events = select.select([pr], [], [], - self._COMMUNICATE_RETRY_TIMEOUT_SECONDS) - except select.error as e: - portage.util.writemsg_level( - "ebuild-ipc: %s: %s\n" % \ - (portage.localization._('during select'), e), - level=logging.ERROR, noiselevel=-1) - continue + fifo_writer.start() + eof = fifo_writer.poll() is not None - if events[0]: - break + while not eof: + fifo_writer._wait_loop(timeout=self._COMMUNICATE_RETRY_TIMEOUT_MS) - if self._daemon_is_alive(): + eof = fifo_writer.poll() is not None + if eof: + break + elif self._daemon_is_alive(): self._timeout_retry_msg(start_time, msg) else: + fifo_writer.cancel() self._no_daemon_msg() - try: - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - except OSError as e: - portage.util.writemsg_level( - "ebuild-ipc: %s\n" % (e,), - level=logging.ERROR, noiselevel=-1) + fifo_writer.wait() return 2 - try: - wait_retval = os.waitpid(pid, 0) - except OSError as e: - portage.util.writemsg_level( - "ebuild-ipc: %s: %s\n" % (msg, e), - level=logging.ERROR, noiselevel=-1) - return 2 + return fifo_writer.wait() - if not os.WIFEXITED(wait_retval[1]): - portage.util.writemsg_level( - "ebuild-ipc: %s: %s\n" % (msg, - portage.localization._('subprocess failure: %s') % \ - wait_retval[1]), - level=logging.ERROR, noiselevel=-1) - return 2 + def _receive_reply(self, input_fd): - return os.WEXITSTATUS(wait_retval[1]) + start_time = time.time() - def _receive_reply(self, input_fd): + pipe_reader = PipeReader(input_files={"input_fd":input_fd}, + scheduler=global_event_loop()) + pipe_reader.start() - # Timeouts are handled by the parent process, so just - # block until input is available. For maximum portability, - # use a single atomic read. - buf = None - while True: - try: - events = select.select([input_fd], [], []) - except select.error as e: - portage.util.writemsg_level( - "ebuild-ipc: %s: %s\n" % \ - (portage.localization._('during select for read'), e), - level=logging.ERROR, noiselevel=-1) - continue - - if events[0]: - # For maximum portability, use os.read() here since - # array.fromfile() and file.read() are both known to - # erroneously return an empty string from this - # non-blocking fifo stream on FreeBSD (bug #337465). - try: - buf = os.read(input_fd, self._BUFSIZE) - except OSError as e: - if e.errno != errno.EAGAIN: - portage.util.writemsg_level( - "ebuild-ipc: %s: %s\n" % \ - (portage.localization._('read error'), e), - level=logging.ERROR, noiselevel=-1) - break - # Assume that another event will be generated - # if there's any relevant data. - continue - - # Only one (atomic) read should be necessary. - if buf: - break + eof = pipe_reader.poll() is not None + + while not eof: + pipe_reader._wait_loop(timeout=self._COMMUNICATE_RETRY_TIMEOUT_MS) + eof = pipe_reader.poll() is not None + if not eof: + if self._daemon_is_alive(): + self._timeout_retry_msg(start_time, + portage.localization._('during read')) + else: + pipe_reader.cancel() + self._no_daemon_msg() + return 2 + + buf = pipe_reader.getvalue() retval = 2 @@ -225,32 +203,9 @@ class EbuildIpc(object): # un-interrupted, while the parent handles all timeout # considerations. This helps to avoid possible race conditions # from interference between timeouts and blocking IO operations. - pr, pw = os.pipe() - pid = os.fork() - - if pid == 0: - retval = 2 - try: - os.close(pr) - - # File streams are in unbuffered mode since we do atomic - # read and write of whole pickles. - output_file = open(self.ipc_in_fifo, 'wb', 0) - output_file.write(pickle.dumps(args)) - output_file.close() - retval = os.EX_OK - except SystemExit: - raise - except: - traceback.print_exc() - finally: - os._exit(retval) - - os.close(pw) - msg = portage.localization._('during write') - retval = self._wait(pid, pr, msg) - os.close(pr) + retval = self._run_writer(FifoWriter(buf=pickle.dumps(args), + fifo=self.ipc_in_fifo, scheduler=global_event_loop()), msg) if retval != os.EX_OK: portage.util.writemsg_level( @@ -263,26 +218,7 @@ class EbuildIpc(object): self._no_daemon_msg() return 2 - pr, pw = os.pipe() - pid = os.fork() - - if pid == 0: - retval = 2 - try: - os.close(pr) - retval = self._receive_reply(input_fd) - except SystemExit: - raise - except: - traceback.print_exc() - finally: - os._exit(retval) - - os.close(pw) - retval = self._wait(pid, pr, portage.localization._('during read')) - os.close(pr) - os.close(input_fd) - return retval + return self._receive_reply(input_fd) def ebuild_ipc_main(args): ebuild_ipc = EbuildIpc() diff --git a/portage_with_autodep/bin/ebuild.sh b/portage_with_autodep/bin/ebuild.sh index 2589113..be044e0 100755 --- a/portage_with_autodep/bin/ebuild.sh +++ b/portage_with_autodep/bin/ebuild.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}" @@ -21,21 +21,23 @@ else # in global scope, even though they are completely useless during # the "depend" phase. for x in diropts docompress exeopts get_KV insopts \ - keepdir KV_major KV_micro KV_minor KV_to_int \ + KV_major KV_micro KV_minor KV_to_int \ libopts register_die_hook register_success_hook \ - remove_path_entry set_unless_changed strip_duplicate_slashes \ - unset_unless_changed use_with use_enable ; do + __strip_duplicate_slashes \ + use_with use_enable ; do eval "${x}() { - if has \"\${EAPI:-0}\" 4-python; then + if ___eapi_disallows_helpers_in_global_scope; then die \"\${FUNCNAME}() calls are not allowed in global scope\" fi }" done - # These dummy functions return false in older EAPIs, in order to ensure that + # These dummy functions return false in non-strict EAPIs, in order to ensure that # `use multislot` is false for the "depend" phase. - for x in use useq usev ; do + funcs="use useq usev" + ___eapi_has_usex && funcs+=" usex" + for x in ${funcs} ; do eval "${x}() { - if has \"\${EAPI:-0}\" 4-python; then + if ___eapi_disallows_helpers_in_global_scope; then die \"\${FUNCNAME}() calls are not allowed in global scope\" else return 1 @@ -44,10 +46,16 @@ else done # These functions die because calls to them during the "depend" phase # are considered to be severe QA violations. - for x in best_version has_version portageq ; do + funcs="best_version has_version portageq" + ___eapi_has_master_repositories && funcs+=" master_repositories" + ___eapi_has_repository_path && funcs+=" repository_path" + ___eapi_has_available_eclasses && funcs+=" available_eclasses" + ___eapi_has_eclass_path && funcs+=" eclass_path" + ___eapi_has_license_path && funcs+=" license_path" + for x in ${funcs} ; do eval "${x}() { die \"\${FUNCNAME}() calls are not allowed in global scope\"; }" done - unset x + unset funcs x fi # Don't use sandbox's BASH_ENV for new shells because it does @@ -66,7 +74,7 @@ export PORTAGE_BZIP2_COMMAND=${PORTAGE_BZIP2_COMMAND:-bzip2} # with shell opts (shopts). Ebuilds/eclasses changing shopts should reset them # when they are done. -qa_source() { +__qa_source() { local shopts=$(shopt) OLDIFS="$IFS" local retval source "$@" @@ -79,7 +87,7 @@ qa_source() { return $retval } -qa_call() { +__qa_call() { local shopts=$(shopt) OLDIFS="$IFS" local retval "$@" @@ -102,20 +110,19 @@ unset GZIP BZIP BZIP2 CDPATH GREP_OPTIONS GREP_COLOR GLOBIGNORE [[ $PORTAGE_QUIET != "" ]] && export PORTAGE_QUIET # sandbox support functions; defined prior to profile.bashrc srcing, since the profile might need to add a default exception (/usr/lib64/conftest fex) -_sb_append_var() { +__sb_append_var() { local _v=$1 ; shift local var="SANDBOX_${_v}" - [[ -z $1 || -n $2 ]] && die "Usage: add$(echo ${_v} | \ - LC_ALL=C tr [:upper:] [:lower:]) <colon-delimited list of paths>" + [[ -z $1 || -n $2 ]] && die "Usage: add$(LC_ALL=C tr "[:upper:]" "[:lower:]" <<< "${_v}") <colon-delimited list of paths>" export ${var}="${!var:+${!var}:}$1" } # bash-4 version: # local var="SANDBOX_${1^^}" -# addread() { _sb_append_var ${0#add} "$@" ; } -addread() { _sb_append_var READ "$@" ; } -addwrite() { _sb_append_var WRITE "$@" ; } -adddeny() { _sb_append_var DENY "$@" ; } -addpredict() { _sb_append_var PREDICT "$@" ; } +# addread() { __sb_append_var ${0#add} "$@" ; } +addread() { __sb_append_var READ "$@" ; } +addwrite() { __sb_append_var WRITE "$@" ; } +adddeny() { __sb_append_var DENY "$@" ; } +addpredict() { __sb_append_var PREDICT "$@" ; } addwrite "${PORTAGE_TMPDIR}" addread "/:${PORTAGE_TMPDIR}" @@ -136,19 +143,11 @@ fi # the sandbox is disabled by default except when overridden in the relevant stages export SANDBOX_ON=0 -esyslog() { - # Custom version of esyslog() to take care of the "Red Star" bug. - # MUST follow functions.sh to override the "" parameter problem. - return 0 -} - # Ensure that $PWD is sane whenever possible, to protect against # exploitation of insecure search path for python -c in ebuilds. -# See bug #239560. -if ! has "$EBUILD_PHASE" clean cleanrm depend help ; then - cd "$PORTAGE_BUILDDIR" || \ - die "PORTAGE_BUILDDIR does not exist: '$PORTAGE_BUILDDIR'" -fi +# See bug #239560 and bug #469338. +cd "${PORTAGE_PYM_PATH}" || \ + die "PORTAGE_PYM_PATH does not exist: '${PORTAGE_PYM_PATH}'" #if no perms are specified, dirs/files will have decent defaults #(not secretive, but not stupid) @@ -178,8 +177,8 @@ debug-print() { # default target printf '%s\n' "${@}" >> "${T}/eclass-debug.log" # let the portage user own/write to this file - chgrp portage "${T}/eclass-debug.log" &>/dev/null - chmod g+w "${T}/eclass-debug.log" &>/dev/null + chgrp "${PORTAGE_GRPNAME:-portage}" "${T}/eclass-debug.log" + chmod g+w "${T}/eclass-debug.log" fi } @@ -208,8 +207,9 @@ inherit() { | fmt -w 75 | while read -r ; do eqawarn "$REPLY" ; done fi + local repo_location local location - local olocation + local potential_location local x # These variables must be restored before returning. @@ -221,9 +221,10 @@ inherit() { local B_DEPEND local B_RDEPEND local B_PDEPEND + local B_HDEPEND while [ "$1" ]; do - location="${ECLASSDIR}/${1}.eclass" - olocation="" + location="" + potential_location="" export ECLASS="$1" __export_funcs_var=__export_functions_$ECLASS_DEPTH @@ -244,43 +245,36 @@ inherit() { fi fi - # any future resolution code goes here - if [ -n "$PORTDIR_OVERLAY" ]; then - local overlay - for overlay in ${PORTDIR_OVERLAY}; do - olocation="${overlay}/eclass/${1}.eclass" - if [ -e "$olocation" ]; then - location="${olocation}" - debug-print " eclass exists: ${location}" - fi - done - fi + for repo_location in "${PORTAGE_ECLASS_LOCATIONS[@]}"; do + potential_location="${repo_location}/eclass/${1}.eclass" + if [[ -f ${potential_location} ]]; then + location="${potential_location}" + debug-print " eclass exists: ${location}" + break + fi + done debug-print "inherit: $1 -> $location" - [ ! -e "$location" ] && die "${1}.eclass could not be found by inherit()" + [[ -z ${location} ]] && die "${1}.eclass could not be found by inherit()" - if [ "${location}" == "${olocation}" ] && \ - ! has "${location}" ${EBUILD_OVERLAY_ECLASSES} ; then - EBUILD_OVERLAY_ECLASSES="${EBUILD_OVERLAY_ECLASSES} ${location}" - fi - - #We need to back up the value of DEPEND and RDEPEND to B_DEPEND and B_RDEPEND + #We need to back up the values of *DEPEND to B_*DEPEND #(if set).. and then restore them after the inherit call. #turn off glob expansion set -f # Retain the old data and restore it later. - unset B_IUSE B_REQUIRED_USE B_DEPEND B_RDEPEND B_PDEPEND + unset B_IUSE B_REQUIRED_USE B_DEPEND B_RDEPEND B_PDEPEND B_HDEPEND [ "${IUSE+set}" = set ] && B_IUSE="${IUSE}" [ "${REQUIRED_USE+set}" = set ] && B_REQUIRED_USE="${REQUIRED_USE}" [ "${DEPEND+set}" = set ] && B_DEPEND="${DEPEND}" [ "${RDEPEND+set}" = set ] && B_RDEPEND="${RDEPEND}" [ "${PDEPEND+set}" = set ] && B_PDEPEND="${PDEPEND}" - unset IUSE REQUIRED_USE DEPEND RDEPEND PDEPEND + [ "${HDEPEND+set}" = set ] && B_HDEPEND="${HDEPEND}" + unset IUSE REQUIRED_USE DEPEND RDEPEND PDEPEND HDEPEND #turn on glob expansion set +f - qa_source "$location" || die "died sourcing $location in inherit()" + __qa_source "$location" || die "died sourcing $location in inherit()" #turn off glob expansion set -f @@ -292,6 +286,7 @@ inherit() { [ "${DEPEND+set}" = set ] && E_DEPEND+="${E_DEPEND:+ }${DEPEND}" [ "${RDEPEND+set}" = set ] && E_RDEPEND+="${E_RDEPEND:+ }${RDEPEND}" [ "${PDEPEND+set}" = set ] && E_PDEPEND+="${E_PDEPEND:+ }${PDEPEND}" + [ "${HDEPEND+set}" = set ] && E_HDEPEND+="${E_HDEPEND:+ }${HDEPEND}" [ "${B_IUSE+set}" = set ] && IUSE="${B_IUSE}" [ "${B_IUSE+set}" = set ] || unset IUSE @@ -308,6 +303,9 @@ inherit() { [ "${B_PDEPEND+set}" = set ] && PDEPEND="${B_PDEPEND}" [ "${B_PDEPEND+set}" = set ] || unset PDEPEND + [ "${B_HDEPEND+set}" = set ] && HDEPEND="${B_HDEPEND}" + [ "${B_HDEPEND+set}" = set ] || unset HDEPEND + #turn on glob expansion set +f @@ -348,7 +346,7 @@ EXPORT_FUNCTIONS() { PORTAGE_BASHRCS_SOURCED=0 -# @FUNCTION: source_all_bashrcs +# @FUNCTION: __source_all_bashrcs # @DESCRIPTION: # Source a relevant bashrc files and perform other miscellaneous # environment initialization when appropriate. @@ -359,7 +357,7 @@ PORTAGE_BASHRCS_SOURCED=0 # * A "default" function which is an alias for the default phase # function for the current phase. # -source_all_bashrcs() { +__source_all_bashrcs() { [[ $PORTAGE_BASHRCS_SOURCED = 1 ]] && return 0 PORTAGE_BASHRCS_SOURCED=1 local x @@ -373,7 +371,7 @@ source_all_bashrcs() { local path_array=($PROFILE_PATHS) restore_IFS for x in "${path_array[@]}" ; do - [ -f "$x/profile.bashrc" ] && qa_source "$x/profile.bashrc" + [ -f "$x/profile.bashrc" ] && __qa_source "$x/profile.bashrc" done fi @@ -390,7 +388,7 @@ source_all_bashrcs() { if [[ $EBUILD_PHASE != depend ]] ; then # The user's bashrc is the ONLY non-portage bit of code that can # change shopts without a QA violation. - for x in "${PM_EBUILD_HOOK_DIR}"/${CATEGORY}/{${PN},${PN}:${SLOT},${P},${PF}}; do + for x in "${PM_EBUILD_HOOK_DIR}"/${CATEGORY}/{${PN},${PN}:${SLOT%/*},${P},${PF}}; do if [ -r "${x}" ]; then # If $- contains x, then tracing has already been enabled # elsewhere for some reason. We preserve it's state so as @@ -470,7 +468,7 @@ if [[ -n ${QA_INTERCEPTORS} ]] ; then fi # Subshell/helper die support (must export for the die helper). -export EBUILD_MASTER_PID=$BASHPID +export EBUILD_MASTER_PID=${BASHPID:-$(__bashpid)} trap 'exit 1' SIGTERM if ! has "$EBUILD_PHASE" clean cleanrm depend && \ @@ -479,7 +477,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ # may have come from another version of ebuild.sh or something. # In any case, preprocess it to prevent any potential interference. # NOTE: export ${FOO}=... requires quoting, unlike normal exports - preprocess_ebuild_env || \ + __preprocess_ebuild_env || \ die "error processing environment" # Colon separated SANDBOX_* variables need to be cumulative. for x in SANDBOX_DENY SANDBOX_READ SANDBOX_PREDICT SANDBOX_WRITE ; do @@ -512,17 +510,22 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ [[ -n $EAPI ]] || EAPI=0 fi -if has "${EAPI:-0}" 4-python; then +if ___eapi_enables_globstar; then shopt -s globstar fi +# Convert quoted paths to array. +eval "PORTAGE_ECLASS_LOCATIONS=(${PORTAGE_ECLASS_LOCATIONS})" + +# Source the ebuild every time for FEATURES=noauto, so that ebuild +# modifications take effect immediately. if ! has "$EBUILD_PHASE" clean cleanrm ; then if [[ $EBUILD_PHASE = depend || ! -f $T/environment || \ - -f $PORTAGE_BUILDDIR/.ebuild_changed ]] || \ - has noauto $FEATURES ; then + -f $PORTAGE_BUILDDIR/.ebuild_changed || \ + " ${FEATURES} " == *" noauto "* ]] ; then # The bashrcs get an opportunity here to set aliases that will be expanded # during sourcing of ebuilds and eclasses. - source_all_bashrcs + __source_all_bashrcs # When EBUILD_PHASE != depend, INHERITED comes pre-initialized # from cache. In order to make INHERITED content independent of @@ -534,8 +537,9 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then # In order to ensure correct interaction between ebuilds and # eclasses, they need to be unset before this process of # interaction begins. - unset EAPI DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ - ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND + unset EAPI DEPEND RDEPEND PDEPEND HDEPEND INHERITED IUSE REQUIRED_USE \ + ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND \ + E_HDEPEND if [[ $PORTAGE_DEBUG != 1 || ${-/x/} != $- ]] ; then source "$EBUILD" || die "error sourcing ebuild" @@ -553,7 +557,10 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then [ "${EAPI+set}" = set ] || EAPI=0 - if has "$EAPI" 0 1 2 3 3_pre2 ; then + # export EAPI for helpers (especially since we unset it above) + export EAPI + + if ___eapi_has_RDEPEND_DEPEND_fallback; then export RDEPEND=${RDEPEND-${DEPEND}} debug-print "RDEPEND: not set... Setting to: ${DEPEND}" fi @@ -563,19 +570,20 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then DEPEND+="${DEPEND:+ }${E_DEPEND}" RDEPEND+="${RDEPEND:+ }${E_RDEPEND}" PDEPEND+="${PDEPEND:+ }${E_PDEPEND}" + HDEPEND+="${HDEPEND:+ }${E_HDEPEND}" REQUIRED_USE+="${REQUIRED_USE:+ }${E_REQUIRED_USE}" - unset ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND \ + unset ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND E_HDEPEND \ __INHERITED_QA_CACHE # alphabetically ordered by $EBUILD_PHASE value - case "$EAPI" in + case ${EAPI} in 0|1) _valid_phases="src_compile pkg_config pkg_info src_install pkg_nofetch pkg_postinst pkg_postrm pkg_preinst pkg_prerm pkg_setup src_test src_unpack" ;; - 2|3|3_pre2) + 2|3) _valid_phases="src_compile pkg_config src_configure pkg_info src_install pkg_nofetch pkg_postinst pkg_postrm pkg_preinst src_prepare pkg_prerm pkg_setup src_test src_unpack" @@ -667,9 +675,13 @@ if [[ $EBUILD_PHASE = depend ]] ; then auxdbkeys="DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND PROVIDE EAPI - PROPERTIES DEFINED_PHASES UNUSED_05 UNUSED_04 + PROPERTIES DEFINED_PHASES HDEPEND UNUSED_04 UNUSED_03 UNUSED_02 UNUSED_01" + if ! ___eapi_has_HDEPEND; then + unset HDEPEND + fi + # The extra $(echo) commands remove newlines. if [ -n "${dbkey}" ] ; then > "${dbkey}" @@ -678,31 +690,28 @@ if [[ $EBUILD_PHASE = depend ]] ; then done else for f in ${auxdbkeys} ; do - echo $(echo ${!f}) 1>&9 || exit $? + eval "echo \$(echo \${!f}) 1>&${PORTAGE_PIPE_FD}" || exit $? done - exec 9>&- + eval "exec ${PORTAGE_PIPE_FD}>&-" fi set +f else - # Note: readonly variables interfere with preprocess_ebuild_env(), so + # Note: readonly variables interfere with __preprocess_ebuild_env(), so # declare them only after it has already run. declare -r $PORTAGE_READONLY_METADATA $PORTAGE_READONLY_VARS - case "$EAPI" in - 0|1|2) - [[ " ${FEATURES} " == *" force-prefix "* ]] && \ - declare -r ED EPREFIX EROOT - ;; - *) - declare -r ED EPREFIX EROOT - ;; - esac + if ___eapi_has_prefix_variables; then + declare -r ED EPREFIX EROOT + fi if [[ -n $EBUILD_SH_ARGS ]] ; then ( # Don't allow subprocesses to inherit the pipe which # emerge uses to monitor ebuild.sh. - exec 9>&- - ebuild_main ${EBUILD_SH_ARGS} + if [[ -n ${PORTAGE_PIPE_FD} ]] ; then + eval "exec ${PORTAGE_PIPE_FD}>&-" + unset PORTAGE_PIPE_FD + fi + __ebuild_main ${EBUILD_SH_ARGS} exit 0 ) exit $? diff --git a/portage_with_autodep/bin/egencache b/portage_with_autodep/bin/egencache index 2f53b40..9b22363 100755 --- a/portage_with_autodep/bin/egencache +++ b/portage_with_autodep/bin/egencache @@ -1,15 +1,17 @@ #!/usr/bin/python -# Copyright 2009-2012 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +# unicode_literals for compat with TextIOWrapper in Python 2 +from __future__ import print_function, unicode_literals +import platform import signal import sys # This block ensures that ^C interrupts are handled quietly. try: - def exithandler(signum,frame): + def exithandler(signum, _frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) sys.exit(128 + signum) @@ -20,26 +22,39 @@ try: except KeyboardInterrupt: sys.exit(128 + signal.SIGINT) +def debug_signal(_signum, _frame): + import pdb + pdb.set_trace() + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) + import io import logging -import optparse import subprocess import time import textwrap import re -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os, _encodings, _unicode_encode, _unicode_decode from _emerge.MetadataRegen import MetadataRegen from portage.cache.cache_errors import CacheError, StatCollision +from portage.const import TIMESTAMP_FORMAT from portage.manifest import guessManifestFileType +from portage.package.ebuild._parallel_manifest.ManifestScheduler import ManifestScheduler from portage.util import cmp_sort_key, writemsg_level +from portage.util._argparse import ArgumentParser +from portage.util._async.run_main_scheduler import run_main_scheduler +from portage.util._eventloop.global_event_loop import global_event_loop from portage import cpv_getkey from portage.dep import Atom, isjustname from portage.versions import pkgsplit, vercmp @@ -59,72 +74,98 @@ else: from repoman.utilities import FindVCS if sys.hexversion >= 0x3000000: + # pylint: disable=W0622 long = int def parse_args(args): usage = "egencache [options] <action> ... [atom] ..." - parser = optparse.OptionParser(usage=usage) + parser = ArgumentParser(usage=usage) - actions = optparse.OptionGroup(parser, 'Actions') - actions.add_option("--update", + actions = parser.add_argument_group('Actions') + actions.add_argument("--update", action="store_true", - help="update metadata/cache/ (generate as necessary)") - actions.add_option("--update-use-local-desc", + help="update metadata/md5-cache/ (generate as necessary)") + actions.add_argument("--update-use-local-desc", action="store_true", help="update the use.local.desc file from metadata.xml") - actions.add_option("--update-changelogs", + actions.add_argument("--update-changelogs", action="store_true", help="update the ChangeLog files from SCM logs") - parser.add_option_group(actions) + actions.add_argument("--update-manifests", + action="store_true", + help="update manifests") - common = optparse.OptionGroup(parser, 'Common options') - common.add_option("--repo", + common = parser.add_argument_group('Common options') + common.add_argument("--repo", action="store", - help="name of repo to operate on (default repo is located at $PORTDIR)") - common.add_option("--config-root", + help="name of repo to operate on") + common.add_argument("--config-root", help="location of portage config files", dest="portage_configroot") - common.add_option("--portdir", - help="override the portage tree location", + common.add_argument("--gpg-dir", + help="override the PORTAGE_GPG_DIR variable", + dest="gpg_dir") + common.add_argument("--gpg-key", + help="override the PORTAGE_GPG_KEY variable", + dest="gpg_key") + common.add_argument("--portdir", + help="override the PORTDIR variable (deprecated in favor of --repositories-configuration)", dest="portdir") - common.add_option("--portdir-overlay", - help="override the PORTDIR_OVERLAY variable (requires that --repo is also specified)", + common.add_argument("--portdir-overlay", + help="override the PORTDIR_OVERLAY variable (deprecated in favor of --repositories-configuration)", dest="portdir_overlay") - common.add_option("--tolerant", + common.add_argument("--repositories-configuration", + help="override configuration of repositories (in format of repos.conf)", + dest="repositories_configuration") + common.add_argument("--sign-manifests", + choices=('y', 'n'), + metavar="<y|n>", + help="manually override layout.conf sign-manifests setting") + common.add_argument("--strict-manifests", + choices=('y', 'n'), + metavar="<y|n>", + help="manually override \"strict\" FEATURES setting") + common.add_argument("--thin-manifests", + choices=('y', 'n'), + metavar="<y|n>", + help="manually override layout.conf thin-manifests setting") + common.add_argument("--tolerant", action="store_true", help="exit successfully if only minor errors occurred") - common.add_option("--ignore-default-opts", + common.add_argument("--ignore-default-opts", action="store_true", help="do not use the EGENCACHE_DEFAULT_OPTS environment variable") - parser.add_option_group(common) + common.add_argument("--write-timestamp", + action="store_true", + help="write metadata/timestamp.chk as required for rsync repositories") - update = optparse.OptionGroup(parser, '--update options') - update.add_option("--cache-dir", + update = parser.add_argument_group('--update options') + update.add_argument("--cache-dir", help="location of the metadata cache", dest="cache_dir") - update.add_option("--jobs", + update.add_argument("-j", "--jobs", + type=int, action="store", help="max ebuild processes to spawn") - update.add_option("--load-average", + update.add_argument("--load-average", + type=float, action="store", help="max load allowed when spawning multiple jobs", dest="load_average") - update.add_option("--rsync", + update.add_argument("--rsync", action="store_true", help="enable rsync stat collision workaround " + \ "for bug 139134 (use with --update)") - parser.add_option_group(update) - uld = optparse.OptionGroup(parser, '--update-use-local-desc options') - uld.add_option("--preserve-comments", + uld = parser.add_argument_group('--update-use-local-desc options') + uld.add_argument("--preserve-comments", action="store_true", help="preserve the comments from the existing use.local.desc file") - uld.add_option("--use-local-desc-output", + uld.add_argument("--use-local-desc-output", help="output file for use.local.desc data (or '-' for stdout)", dest="uld_output") - parser.add_option_group(uld) - options, args = parser.parse_args(args) + options, args = parser.parse_known_args(args) if options.jobs: jobs = None @@ -171,9 +212,12 @@ def parse_args(args): parser.error("Write access denied: --cache-dir='%s'" % \ (options.cache_dir,)) - if options.portdir_overlay is not None and \ - options.repo is None: - parser.error("--portdir-overlay option requires --repo option") + if options.portdir is not None: + writemsg_level("egencache: warning: --portdir option is deprecated in favor of --repositories-configuration option\n", + level=logging.WARNING, noiselevel=-1) + if options.portdir_overlay is not None: + writemsg_level("egencache: warning: --portdir-overlay option is deprecated in favor of --repositories-configuration option\n", + level=logging.WARNING, noiselevel=-1) for atom in args: try: @@ -215,9 +259,11 @@ class GenCache(object): else: self._cp_set = None self._cp_missing = set() + write_auxdb = "metadata-transfer" in portdb.settings.features self._regen = MetadataRegen(portdb, cp_iter=cp_iter, consumer=self._metadata_callback, - max_jobs=max_jobs, max_load=max_load) + max_jobs=max_jobs, max_load=max_load, + write_auxdb=write_auxdb, main=True) self.returncode = os.EX_OK conf = portdb.repositories.get_repo_for_location(tree) self._trg_caches = tuple(conf.iter_pregenerated_caches( @@ -255,98 +301,74 @@ class GenCache(object): def _write_cache(self, trg_cache, cpv, repo_path, metadata, ebuild_hash): - if not hasattr(trg_cache, 'raise_stat_collision'): - # This cache does not avoid redundant writes automatically, - # so check for an identical existing entry before writing. - # This prevents unnecessary disk writes and can also prevent - # unnecessary rsync transfers. - try: - dest = trg_cache[cpv] - except (KeyError, CacheError): - pass - else: - if trg_cache.validate_entry(dest, - ebuild_hash, self._eclass_db): - identical = True - for k in self._auxdbkeys: - if dest.get(k, '') != metadata.get(k, ''): - identical = False - break - if identical: - return + if not hasattr(trg_cache, 'raise_stat_collision'): + # This cache does not avoid redundant writes automatically, + # so check for an identical existing entry before writing. + # This prevents unnecessary disk writes and can also prevent + # unnecessary rsync transfers. + try: + dest = trg_cache[cpv] + except (KeyError, CacheError): + pass + else: + if trg_cache.validate_entry(dest, + ebuild_hash, self._eclass_db): + identical = True + for k in self._auxdbkeys: + if dest.get(k, '') != metadata.get(k, ''): + identical = False + break + if identical: + return + try: + chf = trg_cache.validation_chf + metadata['_%s_' % chf] = getattr(ebuild_hash, chf) try: - chf = trg_cache.validation_chf - metadata['_%s_' % chf] = getattr(ebuild_hash, chf) + trg_cache[cpv] = metadata + except StatCollision as sc: + # If the content of a cache entry changes and neither the + # file mtime nor size changes, it will prevent rsync from + # detecting changes. Cache backends may raise this + # exception from _setitem() if they detect this type of stat + # collision. These exceptions are handled by bumping the + # mtime on the ebuild (and the corresponding cache entry). + # See bug #139134. It is convenient to include checks for + # redundant writes along with the internal StatCollision + # detection code, so for caches with the + # raise_stat_collision attribute, we do not need to + # explicitly check for redundant writes like we do for the + # other cache types above. + max_mtime = sc.mtime + for _ec, ec_hash in metadata['_eclasses_'].items(): + if max_mtime < ec_hash.mtime: + max_mtime = ec_hash.mtime + if max_mtime == sc.mtime: + max_mtime += 1 + max_mtime = long(max_mtime) try: + os.utime(ebuild_hash.location, (max_mtime, max_mtime)) + except OSError as e: + self.returncode |= 1 + writemsg_level( + "%s writing target: %s\n" % (cpv, e), + level=logging.ERROR, noiselevel=-1) + else: + ebuild_hash.mtime = max_mtime + metadata['_mtime_'] = max_mtime trg_cache[cpv] = metadata - except StatCollision as sc: - # If the content of a cache entry changes and neither the - # file mtime nor size changes, it will prevent rsync from - # detecting changes. Cache backends may raise this - # exception from _setitem() if they detect this type of stat - # collision. These exceptions are handled by bumping the - # mtime on the ebuild (and the corresponding cache entry). - # See bug #139134. It is convenient to include checks for - # redundant writes along with the internal StatCollision - # detection code, so for caches with the - # raise_stat_collision attribute, we do not need to - # explicitly check for redundant writes like we do for the - # other cache types above. - max_mtime = sc.mtime - for ec, ec_hash in metadata['_eclasses_'].items(): - if max_mtime < ec_hash.mtime: - max_mtime = ec_hash.mtime - if max_mtime == sc.mtime: - max_mtime += 1 - max_mtime = long(max_mtime) - try: - os.utime(ebuild_hash.location, (max_mtime, max_mtime)) - except OSError as e: - self.returncode |= 1 - writemsg_level( - "%s writing target: %s\n" % (cpv, e), - level=logging.ERROR, noiselevel=-1) - else: - ebuild_hash.mtime = max_mtime - metadata['_mtime_'] = max_mtime - trg_cache[cpv] = metadata - self._portdb.auxdb[repo_path][cpv] = metadata + self._portdb.auxdb[repo_path][cpv] = metadata - except CacheError as ce: - self.returncode |= 1 - writemsg_level( - "%s writing target: %s\n" % (cpv, ce), - level=logging.ERROR, noiselevel=-1) + except CacheError as ce: + self.returncode |= 1 + writemsg_level( + "%s writing target: %s\n" % (cpv, ce), + level=logging.ERROR, noiselevel=-1) def run(self): - - received_signal = [] - - def sighandler(signum, frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - self._regen.terminate() - received_signal.append(128 + signum) - - earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler) - earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler) - - try: - self._regen.run() - finally: - # Restore previous handlers - if earlier_sigint_handler is not None: - signal.signal(signal.SIGINT, earlier_sigint_handler) - else: - signal.signal(signal.SIGINT, signal.SIG_DFL) - if earlier_sigterm_handler is not None: - signal.signal(signal.SIGTERM, earlier_sigterm_handler) - else: - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - if received_signal: - sys.exit(received_signal[0]) + signum = run_main_scheduler(self._regen) + if signum is not None: + sys.exit(128 + signum) self.returncode |= self._regen.returncode @@ -371,8 +393,8 @@ class GenCache(object): self.returncode |= 1 writemsg_level( "Error listing cache entries for " + \ - "'%s/metadata/cache': %s, continuing...\n" % \ - (self._portdb.porttree_root, ce), + "'%s': %s, continuing...\n" % \ + (trg_cache.location, ce), level=logging.ERROR, noiselevel=-1) else: @@ -393,8 +415,8 @@ class GenCache(object): self.returncode |= 1 writemsg_level( "Error listing cache entries for " + \ - "'%s/metadata/cache': %s, continuing...\n" % \ - (self._portdb.porttree_root, ce), + "'%s': %s, continuing...\n" % \ + (trg_cache.location, ce), level=logging.ERROR, noiselevel=-1) if cp_missing: @@ -426,6 +448,9 @@ class GenCache(object): "committing target: %s\n" % (ce,), level=logging.ERROR, noiselevel=-1) + if hasattr(trg_cache, '_prune_empty_dirs'): + trg_cache._prune_empty_dirs() + class GenUseLocalDesc(object): def __init__(self, portdb, output=None, preserve_comments=False): @@ -433,7 +458,7 @@ class GenUseLocalDesc(object): self._portdb = portdb self._output = output self._preserve_comments = preserve_comments - + def run(self): repo_path = self._portdb.porttrees[0] ops = {'<':0, '<=':1, '=':2, '>=':3, '>':4} @@ -506,14 +531,14 @@ class GenUseLocalDesc(object): encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['repo.content'], errors='backslashreplace') - output.write(_unicode_decode('\n')) + output.write('\n') else: - output.write(textwrap.dedent(_unicode_decode('''\ + output.write(textwrap.dedent('''\ # This file is deprecated as per GLEP 56 in favor of metadata.xml. Please add # your descriptions to your package's metadata.xml ONLY. # * generated automatically using egencache * - '''))) + ''')) # The cmp function no longer exists in python3, so we'll # implement our own here under a slightly different name @@ -541,7 +566,8 @@ class GenUseLocalDesc(object): for cp in self._portdb.cp_all(): metadata_path = os.path.join(repo_path, cp, 'metadata.xml') try: - metadata = ElementTree.parse(metadata_path, + metadata = ElementTree.parse(_unicode_encode(metadata_path, + encoding=_encodings['fs'], errors='strict'), parser=ElementTree.XMLParser( target=_MetadataTreeBuilder())) except IOError: @@ -597,8 +623,7 @@ class GenUseLocalDesc(object): resatoms = sorted(reskeys, key=cmp_sort_key(atomcmp)) resdesc = resdict[reskeys[resatoms[-1]]] - output.write(_unicode_decode( - '%s:%s - %s\n' % (cp, flag, resdesc))) + output.write('%s:%s - %s\n' % (cp, flag, resdesc)) output.close() @@ -620,7 +645,8 @@ class _special_filename(_filename_base): self.file_name = file_name self.file_type = guessManifestFileType(file_name) - def file_type_lt(self, a, b): + @staticmethod + def file_type_lt(a, b): """ Defines an ordering between file types. """ @@ -695,12 +721,12 @@ class GenChangeLogs(object): self.returncode |= 2 return - output.write(textwrap.dedent(_unicode_decode('''\ + output.write(textwrap.dedent('''\ # ChangeLog for %s # Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2 # $Header: $ - ''' % (cp, time.strftime('%Y'))))) + ''' % (cp, time.strftime('%Y')))) # now grab all the commits commits = self.grab(['git', 'rev-list', 'HEAD', '--', '.']).split() @@ -764,11 +790,10 @@ class GenChangeLogs(object): # Reverse the sort order for headers. for c in reversed(changed): if c.startswith('+') and c.endswith('.ebuild'): - output.write(_unicode_decode( - '*%s (%s)\n' % (c[1:-7], date))) + output.write('*%s (%s)\n' % (c[1:-7], date)) wroteheader = True if wroteheader: - output.write(_unicode_decode('\n')) + output.write('\n') # strip '<cp>: ', '[<cp>] ', and similar body[0] = re.sub(r'^\W*' + re.escape(cp) + r'\W+', '', body[0]) @@ -788,13 +813,12 @@ class GenChangeLogs(object): # don't break filenames on hyphens self._wrapper.break_on_hyphens = False - output.write(_unicode_decode( - self._wrapper.fill( - '%s; %s %s:' % (date, author, ', '.join(changed))))) + output.write(self._wrapper.fill( + '%s; %s %s:' % (date, author, ', '.join(changed)))) # but feel free to break commit messages there self._wrapper.break_on_hyphens = True - output.write(_unicode_decode( - '\n%s\n\n' % '\n'.join(self._wrapper.fill(x) for x in body))) + output.write( + '\n%s\n\n' % '\n'.join(self._wrapper.fill(x) for x in body)) output.close() @@ -827,17 +851,22 @@ class GenChangeLogs(object): self.generate_changelog(cp) def egencache_main(args): - parser, options, atoms = parse_args(args) - - config_root = options.config_root # The calling environment is ignored, so the program is # completely controlled by commandline arguments. env = {} - if options.repo is None: - env['PORTDIR_OVERLAY'] = '' - elif options.portdir_overlay: + if not sys.stdout.isatty(): + portage.output.nocolor() + env['NOCOLOR'] = 'true' + + parser, options, atoms = parse_args(args) + + config_root = options.config_root + + if options.repositories_configuration is not None: + env['PORTAGE_REPOSITORIES'] = options.repositories_configuration + elif options.portdir_overlay is not None: env['PORTDIR_OVERLAY'] = options.portdir_overlay if options.cache_dir is not None: @@ -851,7 +880,8 @@ def egencache_main(args): default_opts = None if not options.ignore_default_opts: - default_opts = settings.get('EGENCACHE_DEFAULT_OPTS', '').split() + default_opts = portage.util.shlex_split( + settings.get('EGENCACHE_DEFAULT_OPTS', '')) if default_opts: parser, options, args = parse_args(default_opts + args) @@ -862,18 +892,50 @@ def egencache_main(args): settings = portage.config(config_root=config_root, local_config=False, env=env) - if not options.update and not options.update_use_local_desc \ - and not options.update_changelogs: + if not (options.update or options.update_use_local_desc or + options.update_changelogs or options.update_manifests): parser.error('No action specified') return 1 + if options.repo is None: + if len(settings.repositories.prepos) == 2: + for repo in settings.repositories: + if repo.name != "DEFAULT": + options.repo = repo.name + break + + if options.repo is None: + parser.error("--repo option is required") + + repo_path = settings.repositories.treemap.get(options.repo) + if repo_path is None: + parser.error("Unable to locate repository named '%s'" % (options.repo,)) + return 1 + + repo_config = settings.repositories.get_repo_for_location(repo_path) + + if options.strict_manifests is not None: + if options.strict_manifests == "y": + settings.features.add("strict") + else: + settings.features.discard("strict") + if options.update and 'metadata-transfer' not in settings.features: - settings.features.add('metadata-transfer') + # Forcibly enable metadata-transfer if portdbapi has a pregenerated + # cache that does not support eclass validation. + cache = repo_config.get_pregenerated_cache( + portage.dbapi.dbapi._known_keys, readonly=True) + if cache is not None and not cache.complete_eclass_entries: + settings.features.add('metadata-transfer') + cache = None settings.lock() portdb = portage.portdbapi(mysettings=settings) + # Limit ebuilds to the specified repo. + portdb.porttrees = [repo_path] + if options.update: if options.cache_dir is not None: # already validated earlier @@ -889,17 +951,71 @@ def egencache_main(args): level=logging.ERROR, noiselevel=-1) return 1 - if options.repo is not None: - repo_path = portdb.getRepositoryPath(options.repo) - if repo_path is None: - parser.error("Unable to locate repository named '%s'" % \ - (options.repo,)) - return 1 + if options.sign_manifests is not None: + repo_config.sign_manifest = options.sign_manifests == 'y' - # Limit ebuilds to the specified repo. - portdb.porttrees = [repo_path] - else: - portdb.porttrees = [portdb.porttree_root] + if options.thin_manifests is not None: + repo_config.thin_manifest = options.thin_manifests == 'y' + + gpg_cmd = None + gpg_vars = None + force_sign_key = None + + if options.update_manifests: + if repo_config.sign_manifest: + + sign_problem = False + gpg_dir = None + gpg_cmd = settings.get("PORTAGE_GPG_SIGNING_COMMAND") + if gpg_cmd is None: + writemsg_level("egencache: error: " + "PORTAGE_GPG_SIGNING_COMMAND is unset! " + "Is make.globals missing?\n", + level=logging.ERROR, noiselevel=-1) + sign_problem = True + elif "${PORTAGE_GPG_KEY}" in gpg_cmd and \ + options.gpg_key is None and \ + "PORTAGE_GPG_KEY" not in settings: + writemsg_level("egencache: error: " + "PORTAGE_GPG_KEY is unset!\n", + level=logging.ERROR, noiselevel=-1) + sign_problem = True + elif "${PORTAGE_GPG_DIR}" in gpg_cmd: + if options.gpg_dir is not None: + gpg_dir = options.gpg_dir + elif "PORTAGE_GPG_DIR" not in settings: + gpg_dir = os.path.expanduser("~/.gnupg") + else: + gpg_dir = os.path.expanduser(settings["PORTAGE_GPG_DIR"]) + if not os.access(gpg_dir, os.X_OK): + writemsg_level(("egencache: error: " + "Unable to access directory: " + "PORTAGE_GPG_DIR='%s'\n") % gpg_dir, + level=logging.ERROR, noiselevel=-1) + sign_problem = True + + if sign_problem: + writemsg_level("egencache: You may disable manifest " + "signatures with --sign-manifests=n or by setting " + "\"sign-manifests = false\" in metadata/layout.conf\n", + level=logging.ERROR, noiselevel=-1) + return 1 + + gpg_vars = {} + if gpg_dir is not None: + gpg_vars["PORTAGE_GPG_DIR"] = gpg_dir + gpg_var_names = [] + if options.gpg_key is None: + gpg_var_names.append("PORTAGE_GPG_KEY") + else: + gpg_vars["PORTAGE_GPG_KEY"] = options.gpg_key + + for k in gpg_var_names: + v = settings.get(k) + if v is not None: + gpg_vars[k] = v + + force_sign_key = gpg_vars.get("PORTAGE_GPG_KEY") ret = [os.EX_OK] @@ -918,6 +1034,29 @@ def egencache_main(args): else: ret.append(gen_cache.returncode) + if options.update_manifests: + + cp_iter = None + if atoms: + cp_iter = iter(atoms) + + event_loop = global_event_loop() + scheduler = ManifestScheduler(portdb, cp_iter=cp_iter, + gpg_cmd=gpg_cmd, gpg_vars=gpg_vars, + force_sign_key=force_sign_key, + max_jobs=options.jobs, + max_load=options.load_average, + event_loop=event_loop) + + signum = run_main_scheduler(scheduler) + if signum is not None: + sys.exit(128 + signum) + + if options.tolerant: + ret.append(os.EX_OK) + else: + ret.append(scheduler.returncode) + if options.update_use_local_desc: gen_desc = GenUseLocalDesc(portdb, output=options.uld_output, @@ -930,6 +1069,16 @@ def egencache_main(args): gen_clogs.run() ret.append(gen_clogs.returncode) + if options.write_timestamp: + timestamp_path = os.path.join(repo_path, 'metadata', 'timestamp.chk') + try: + with open(timestamp_path, 'w') as f: + f.write(time.strftime('%s\n' % TIMESTAMP_FORMAT, time.gmtime())) + except IOError: + ret.append(os.EX_IOERR) + else: + ret.append(os.EX_OK) + return max(ret) if __name__ == "__main__": diff --git a/portage_with_autodep/bin/emaint b/portage_with_autodep/bin/emaint index 1bee0fe..adf44d0 100755 --- a/portage_with_autodep/bin/emaint +++ b/portage_with_autodep/bin/emaint @@ -1,654 +1,42 @@ #!/usr/bin/python -O -# vim: noet : +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""System health checks and maintenance utilities. +""" from __future__ import print_function -import errno -import re -import signal -import stat import sys -import textwrap -import time -from optparse import OptionParser, OptionValueError - +import errno +# This block ensures that ^C interrupts are handled quietly. try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - -from portage import os -from portage.util import writemsg - -if sys.hexversion >= 0x3000000: - long = int - -class WorldHandler(object): - - short_desc = "Fix problems in the world file" - - def name(): - return "world" - name = staticmethod(name) - - def __init__(self): - self.invalid = [] - self.not_installed = [] - self.invalid_category = [] - self.okay = [] - from portage._sets import load_default_config - setconfig = load_default_config(portage.settings, - portage.db[portage.settings['EROOT']]) - self._sets = setconfig.getSets() - - def _check_world(self, onProgress): - categories = set(portage.settings.categories) - eroot = portage.settings['EROOT'] - self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) - self.found = os.access(self.world_file, os.R_OK) - vardb = portage.db[eroot]["vartree"].dbapi - - from portage._sets import SETPREFIX - sets = self._sets - world_atoms = list(sets["selected"]) - maxval = len(world_atoms) - if onProgress: - onProgress(maxval, 0) - for i, atom in enumerate(world_atoms): - if not isinstance(atom, portage.dep.Atom): - if atom.startswith(SETPREFIX): - s = atom[len(SETPREFIX):] - if s in sets: - self.okay.append(atom) - else: - self.not_installed.append(atom) - else: - self.invalid.append(atom) - if onProgress: - onProgress(maxval, i+1) - continue - okay = True - if not vardb.match(atom): - self.not_installed.append(atom) - okay = False - if portage.catsplit(atom.cp)[0] not in categories: - self.invalid_category.append(atom) - okay = False - if okay: - self.okay.append(atom) - if onProgress: - onProgress(maxval, i+1) - - def check(self, onProgress=None): - self._check_world(onProgress) - errors = [] - if self.found: - errors += ["'%s' is not a valid atom" % x for x in self.invalid] - errors += ["'%s' is not installed" % x for x in self.not_installed] - errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category] - else: - errors.append(self.world_file + " could not be opened for reading") - return errors - - def fix(self, onProgress=None): - world_set = self._sets["selected"] - world_set.lock() - try: - world_set.load() # maybe it's changed on disk - before = set(world_set) - self._check_world(onProgress) - after = set(self.okay) - errors = [] - if before != after: - try: - world_set.replace(self.okay) - except portage.exception.PortageException: - errors.append("%s could not be opened for writing" % \ - self.world_file) - return errors - finally: - world_set.unlock() - -class BinhostHandler(object): - - short_desc = "Generate a metadata index for binary packages" - - def name(): - return "binhost" - name = staticmethod(name) - - def __init__(self): - eroot = portage.settings['EROOT'] - self._bintree = portage.db[eroot]["bintree"] - self._bintree.populate() - self._pkgindex_file = self._bintree._pkgindex_file - self._pkgindex = self._bintree._load_pkgindex() - - def _need_update(self, cpv, data): - - if "MD5" not in data: - return True - - size = data.get("SIZE") - if size is None: - return True - - mtime = data.get("MTIME") - if mtime is None: - return True - - pkg_path = self._bintree.getname(cpv) - try: - s = os.lstat(pkg_path) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - # We can't update the index for this one because - # it disappeared. - return False - - try: - if long(mtime) != s[stat.ST_MTIME]: - return True - if long(size) != long(s.st_size): - return True - except ValueError: - return True - - return False - - def check(self, onProgress=None): - missing = [] - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - if onProgress: - onProgress(maxval, i+1) - errors = ["'%s' is not in Packages" % cpv for cpv in missing] - stale = set(metadata).difference(cpv_all) - for cpv in stale: - errors.append("'%s' is not in the repository" % cpv) - return errors - - def fix(self, onProgress=None): - bintree = self._bintree - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - missing = [] - maxval = 0 - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - stale = set(metadata).difference(cpv_all) - if missing or stale: - from portage import locks - pkgindex_lock = locks.lockfile( - self._pkgindex_file, wantnewlockfile=1) - try: - # Repopulate with lock held. - bintree._populate() - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - - pkgindex = bintree._load_pkgindex() - self._pkgindex = pkgindex - - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - # Recount missing packages, with lock held. - del missing[:] - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - maxval = len(missing) - for i, cpv in enumerate(missing): - try: - metadata[cpv] = bintree._pkgindex_entry(cpv) - except portage.exception.InvalidDependString: - writemsg("!!! Invalid binary package: '%s'\n" % \ - bintree.getname(cpv), noiselevel=-1) - - if onProgress: - onProgress(maxval, i+1) - - for cpv in set(metadata).difference( - self._bintree.dbapi.cpv_all()): - del metadata[cpv] - - # We've updated the pkgindex, so set it to - # repopulate when necessary. - bintree.populated = False - - del pkgindex.packages[:] - pkgindex.packages.extend(metadata.values()) - from portage.util import atomic_ofstream - f = atomic_ofstream(self._pkgindex_file) - try: - self._pkgindex.write(f) - finally: - f.close() - finally: - locks.unlockfile(pkgindex_lock) - - if onProgress: - if maxval == 0: - maxval = 1 - onProgress(maxval, maxval) - return None - -class MoveHandler(object): - - def __init__(self, tree, porttree): - self._tree = tree - self._portdb = porttree.dbapi - self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] - self._master_repo = \ - self._portdb.getRepositoryName(self._portdb.porttree_root) - - def _grab_global_updates(self): - from portage.update import grab_updates, parse_updates - retupdates = {} - errors = [] - - for repo_name in self._portdb.getRepositories(): - repo = self._portdb.getRepositoryPath(repo_name) - updpath = os.path.join(repo, "profiles", "updates") - if not os.path.isdir(updpath): - continue - - try: - rawupdates = grab_updates(updpath) - except portage.exception.DirectoryNotFound: - rawupdates = [] - upd_commands = [] - for mykey, mystat, mycontent in rawupdates: - commands, errors = parse_updates(mycontent) - upd_commands.extend(commands) - errors.extend(errors) - retupdates[repo_name] = upd_commands - - if self._master_repo in retupdates: - retupdates['DEFAULT'] = retupdates[self._master_repo] + import signal - return retupdates, errors + def exithandler(signum, _frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + sys.exit(128 + signum) - def check(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - match = self._tree.dbapi.match - aux_get = self._tree.dbapi.aux_get - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) +except KeyboardInterrupt: + sys.exit(1) - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - origcp, newcp = update_cmd[1:] - for cpv in match(origcp): - if repo_match(aux_get(cpv, ["repository"])[0]): - errors.append("'%s' moved to '%s'" % (cpv, newcp)) - elif update_cmd[0] == "slotmove": - pkg, origslot, newslot = update_cmd[1:] - for cpv in match(pkg): - slot, prepo = aux_get(cpv, ["SLOT", "repository"]) - if slot == origslot and repo_match(prepo): - errors.append("'%s' slot moved from '%s' to '%s'" % \ - (cpv, origslot, newslot)) - if onProgress: - onProgress(0, 0) +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True +from portage.emaint.main import emaint_main - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - cpv_all = self._tree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - aux_update = self._tree.dbapi.aux_update - meta_keys = self._update_keys + ['repository'] - from portage.update import update_dbentries - if onProgress: - onProgress(maxval, 0) - for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - repository = metadata.pop('repository') - try: - updates = allupdates[repository] - except KeyError: - try: - updates = allupdates['DEFAULT'] - except KeyError: - continue - if not updates: - continue - metadata_updates = update_dbentries(updates, metadata) - if metadata_updates: - errors.append("'%s' has outdated metadata" % cpv) - if onProgress: - onProgress(maxval, i+1) - return errors - - def fix(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - move = self._tree.dbapi.move_ent - slotmove = self._tree.dbapi.move_slot_ent - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue - - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) - - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - move(update_cmd, repo_match=repo_match) - elif update_cmd[0] == "slotmove": - slotmove(update_cmd, repo_match=repo_match) - if onProgress: - onProgress(0, 0) - - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) - return errors - -class MoveInstalled(MoveHandler): - - short_desc = "Perform package move updates for installed packages" - - def name(): - return "moveinst" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) - -class MoveBinary(MoveHandler): - - short_desc = "Perform package move updates for binary packages" - - def name(): - return "movebin" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) - -class VdbKeyHandler(object): - def name(): - return "vdbkeys" - name = staticmethod(name) - - def __init__(self): - self.list = portage.db[portage.settings["EROOT"]]["vartree"].dbapi.cpv_all() - self.missing = [] - self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"] - - for p in self.list: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - ismissing = True - for k in self.keys: - if os.path.exists(mydir+k): - ismissing = False - break - if ismissing: - self.missing.append(p) - - def check(self): - return ["%s has missing keys" % x for x in self.missing] - - def fix(self): - - errors = [] - - for p in self.missing: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - if not os.access(mydir+"environment.bz2", os.R_OK): - errors.append("Can't access %s" % (mydir+"environment.bz2")) - elif not os.access(mydir, os.W_OK): - errors.append("Can't create files in %s" % mydir) - else: - env = os.popen("bzip2 -dcq "+mydir+"environment.bz2", "r") - envlines = env.read().split("\n") - env.close() - for k in self.keys: - s = [l for l in envlines if l.startswith(k+"=")] - if len(s) > 1: - errors.append("multiple matches for %s found in %senvironment.bz2" % (k, mydir)) - elif len(s) == 0: - s = "" - else: - s = s[0].split("=",1)[1] - s = s.lstrip("$").strip("\'\"") - s = re.sub("(\\\\[nrt])+", " ", s) - s = " ".join(s.split()).strip() - if s != "": - try: - keyfile = open(mydir+os.sep+k, "w") - keyfile.write(s+"\n") - keyfile.close() - except (IOError, OSError) as e: - errors.append("Could not write %s, reason was: %s" % (mydir+k, e)) - - return errors - -class ProgressHandler(object): - def __init__(self): - self.curval = 0 - self.maxval = 0 - self.last_update = 0 - self.min_display_latency = 0.2 - - def onProgress(self, maxval, curval): - self.maxval = maxval - self.curval = curval - cur_time = time.time() - if cur_time - self.last_update >= self.min_display_latency: - self.last_update = cur_time - self.display() - - def display(self): - raise NotImplementedError(self) - -class CleanResume(object): - - short_desc = "Discard emerge --resume merge lists" - - def name(): - return "cleanresume" - name = staticmethod(name) - - def check(self, onProgress=None): - messages = [] - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - d = mtimedb.get(k) - if d is None: - continue - if not isinstance(d, dict): - messages.append("unrecognized resume list: '%s'" % k) - continue - mergelist = d.get("mergelist") - if mergelist is None or not hasattr(mergelist, "__len__"): - messages.append("unrecognized resume list: '%s'" % k) - continue - messages.append("resume list '%s' contains %d packages" % \ - (k, len(mergelist))) - finally: - if onProgress: - onProgress(maxval, i+1) - return messages - - def fix(self, onProgress=None): - delete_count = 0 - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - if mtimedb.pop(k, None) is not None: - delete_count += 1 - finally: - if onProgress: - onProgress(maxval, i+1) - if delete_count: - mtimedb.commit() - -def emaint_main(myargv): - - # Similar to emerge, emaint needs a default umask so that created - # files (such as the world file) have sane permissions. - os.umask(0o22) - - # TODO: Create a system that allows external modules to be added without - # the need for hard coding. - modules = { - "world" : WorldHandler, - "binhost":BinhostHandler, - "moveinst":MoveInstalled, - "movebin":MoveBinary, - "cleanresume":CleanResume - } - - module_names = list(modules) - module_names.sort() - module_names.insert(0, "all") - - def exclusive(option, *args, **kw): - var = kw.get("var", None) - if var is None: - raise ValueError("var not specified to exclusive()") - if getattr(parser, var, ""): - raise OptionValueError("%s and %s are exclusive options" % (getattr(parser, var), option)) - setattr(parser, var, str(option)) - - - usage = "usage: emaint [options] COMMAND" - - desc = "The emaint program provides an interface to system health " + \ - "checks and maintenance. See the emaint(1) man page " + \ - "for additional information about the following commands:" - - usage += "\n\n" - for line in textwrap.wrap(desc, 65): - usage += "%s\n" % line - usage += "\n" - usage += " %s" % "all".ljust(15) + \ - "Perform all supported commands\n" - for m in module_names[1:]: - usage += " %s%s\n" % (m.ljust(15), modules[m].short_desc) - - parser = OptionParser(usage=usage, version=portage.VERSION) - parser.add_option("-c", "--check", help="check for problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.add_option("-f", "--fix", help="attempt to fix problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.action = None - - - (options, args) = parser.parse_args(args=myargv) - if len(args) != 1: - parser.error("Incorrect number of arguments") - if args[0] not in module_names: - parser.error("%s target is not a known target" % args[0]) - - if parser.action: - action = parser.action - else: - print("Defaulting to --check") - action = "-c/--check" - - if args[0] == "all": - tasks = modules.values() - else: - tasks = [modules[args[0]]] - - - if action == "-c/--check": - status = "Checking %s for problems" - func = "check" - else: - status = "Attempting to fix %s" - func = "fix" - - isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() - for task in tasks: - print(status % task.name()) - inst = task() - onProgress = None - if isatty: - progressBar = portage.output.TermProgressBar() - progressHandler = ProgressHandler() - onProgress = progressHandler.onProgress - def display(): - progressBar.set(progressHandler.curval, progressHandler.maxval) - progressHandler.display = display - def sigwinch_handler(signum, frame): - lines, progressBar.term_columns = \ - portage.output.get_term_size() - signal.signal(signal.SIGWINCH, sigwinch_handler) - result = getattr(inst, func)(onProgress=onProgress) - if isatty: - # make sure the final progress is displayed - progressHandler.display() - print() - signal.signal(signal.SIGWINCH, signal.SIG_DFL) - if result: - print() - print("\n".join(result)) - print("\n") - - print("Finished") - -if __name__ == "__main__": +try: emaint_main(sys.argv[1:]) +except IOError as e: + if e.errno == errno.EACCES: + print("\nemaint: Need superuser access") + sys.exit(1) + else: + raise diff --git a/portage_with_autodep/bin/emerge b/portage_with_autodep/bin/emerge index 6f69244..4d9ea5a 100755 --- a/portage_with_autodep/bin/emerge +++ b/portage_with_autodep/bin/emerge @@ -1,66 +1,79 @@ #!/usr/bin/python -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import platform import signal import sys -# This block ensures that ^C interrupts are handled quietly. + +# This block ensures that ^C interrupts are handled quietly. We handle +# KeyboardInterrupt instead of installing a SIGINT handler, since +# exiting from signal handlers intermittently causes python to ignore +# the SystemExit exception with a message like this: +# Exception SystemExit: 130 in <function remove at 0x7fd2146c1320> ignored try: - def exithandler(signum,frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) + def exithandler(signum, _frame): signal.signal(signal.SIGTERM, signal.SIG_IGN) sys.exit(128 + signum) - signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGTERM, exithandler) # Prevent "[Errno 32] Broken pipe" exceptions when # writing to a pipe. signal.signal(signal.SIGPIPE, signal.SIG_DFL) -except KeyboardInterrupt: - sys.exit(128 + signal.SIGINT) + def debug_signal(_signum, _frame): + import pdb + pdb.set_trace() -def debug_signal(signum, frame): - import pdb - pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 + else: + debug_signum = signal.SIGUSR1 + + signal.signal(debug_signum, debug_signal) -try: - from _emerge.main import emerge_main -except ImportError: from os import path as osp - import sys - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) + pym_path = osp.join(osp.dirname(osp.dirname( + osp.realpath(__file__))), "pym") + sys.path.insert(0, pym_path) + import portage + portage._internal_caller = True + portage._disable_legacy_globals() from _emerge.main import emerge_main -if __name__ == "__main__": - import sys - from portage.exception import ParseError, PermissionDenied - try: - retval = emerge_main() - except PermissionDenied as e: - sys.stderr.write("Permission denied: '%s'\n" % str(e)) - sys.exit(e.errno) - except ParseError as e: - sys.stderr.write("%s\n" % str(e)) - sys.exit(1) - except SystemExit: - raise - except Exception: - # If an unexpected exception occurs then we don't want the mod_echo - # output to obscure the traceback, so dump the mod_echo output before - # showing the traceback. - import traceback - tb_str = traceback.format_exc() + if __name__ == "__main__": + from portage.exception import ParseError, PermissionDenied try: - from portage.elog import mod_echo - except ImportError: - pass - else: - mod_echo.finalize() - sys.stderr.write(tb_str) - sys.exit(1) - sys.exit(retval) + retval = emerge_main() + except PermissionDenied as e: + sys.stderr.write("Permission denied: '%s'\n" % str(e)) + sys.exit(e.errno) + except ParseError as e: + sys.stderr.write("%s\n" % str(e)) + sys.exit(1) + except (KeyboardInterrupt, SystemExit): + raise + except Exception: + # If an unexpected exception occurs then we don't want the + # mod_echo output to obscure the traceback, so dump the + # mod_echo output before showing the traceback. + import traceback + tb_str = traceback.format_exc() + try: + from portage.elog import mod_echo + except ImportError: + pass + else: + mod_echo.finalize() + sys.stderr.write(tb_str) + sys.exit(1) + sys.exit(retval) + +except KeyboardInterrupt: + sys.stderr.write("\n\nExiting on signal %(signal)s\n" % + {"signal": signal.SIGINT}) + sys.stderr.flush() + sys.exit(128 + signal.SIGINT) diff --git a/portage_with_autodep/bin/emerge-webrsync b/portage_with_autodep/bin/emerge-webrsync index bfd9aa2..85730a2 100755 --- a/portage_with_autodep/bin/emerge-webrsync +++ b/portage_with_autodep/bin/emerge-webrsync @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author: Karl Trygve Kalleberg <karltk@gentoo.org> # Rewritten from the old, Perl-based emerge-webrsync script @@ -22,9 +22,9 @@ vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; } # Only echo if not in verbose mode nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; } # warning echos -wecho() { echo "${argv0}: warning: $*" 1>&2 ; } +wecho() { echo "${argv0##*/}: warning: $*" 1>&2 ; } # error echos -eecho() { echo "${argv0}: error: $*" 1>&2 ; } +eecho() { echo "${argv0##*/}: error: $*" 1>&2 ; } argv0=$0 @@ -39,23 +39,33 @@ else eecho "could not find 'portageq'; aborting" exit 1 fi -eval $("${portageq}" envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ - PORTAGE_BIN_PATH PORTAGE_GPG_DIR \ - PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \ - SYNC http_proxy ftp_proxy) -DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync" +eval "$("${portageq}" envvar -v DISTDIR EPREFIX FEATURES \ + FETCHCOMMAND GENTOO_MIRRORS \ + PORTAGE_BIN_PATH PORTAGE_CONFIGROOT PORTAGE_GPG_DIR \ + PORTAGE_NICENESS PORTAGE_REPOSITORIES PORTAGE_RSYNC_EXTRA_OPTS \ + PORTAGE_RSYNC_OPTS PORTAGE_TMPDIR \ + USERLAND http_proxy ftp_proxy)" export http_proxy ftp_proxy +source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 + +repo_name=gentoo +repo_location=$(__repo_key "${repo_name}" location) +if [[ -z ${repo_location} ]]; then + eecho "Repository '${repo_name}' not found" + exit 1 +fi +repo_sync_type=$(__repo_key "${repo_name}" sync-type) + # If PORTAGE_NICENESS is overriden via the env then it will # still pass through the portageq call and override properly. if [ -n "${PORTAGE_NICENESS}" ]; then renice $PORTAGE_NICENESS $$ > /dev/null fi -source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 - do_verbose=0 do_debug=0 +keep=false if has webrsync-gpg ${FEATURES} ; then WEBSYNC_VERIFY_SIGNATURE=1 @@ -99,7 +109,9 @@ get_date_part() { get_utc_second_from_string() { local s="$1" if [[ ${USERLAND} == BSD ]] ; then - date -juf "%Y%m%d" "$s" +"%s" + # Specify zeros for the least significant digits, or else those + # digits are inherited from the current system clock time. + date -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%s" else date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s" fi @@ -108,8 +120,8 @@ get_utc_second_from_string() { get_portage_timestamp() { local portage_current_timestamp=0 - if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then - portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" ) + if [ -f "${repo_location}/metadata/timestamp.x" ]; then + portage_current_timestamp=$(cut -f 1 -d " " "${repo_location}/metadata/timestamp.x" ) fi echo "${portage_current_timestamp}" @@ -125,13 +137,18 @@ fetch_file() { elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then opts="--continue-at - $(nvecho -s -f)" else - rm -f "${FILE}" + rm -f "${DISTDIR}/${FILE}" fi - vecho "Fetching file ${FILE} ..." + __vecho "Fetching file ${FILE} ..." # already set DISTDIR= - eval "${FETCHCOMMAND}" ${opts} - [ -s "${FILE}" ] + eval "${FETCHCOMMAND} ${opts}" + if [[ $? -eq 0 && -s ${DISTDIR}/${FILE} ]] ; then + return 0 + else + rm -f "${DISTDIR}/${FILE}" + return 1 + fi } check_file_digest() { @@ -139,10 +156,12 @@ check_file_digest() { local file="$2" local r=1 - vecho "Checking digest ..." + __vecho "Checking digest ..." if type -P md5sum > /dev/null; then - md5sum -c $digest && r=0 + local md5sum_output=$(md5sum "${file}") + local digest_content=$(< "${digest}") + [ "${md5sum_output%%[[:space:]]*}" = "${digest_content%%[[:space:]]*}" ] && r=0 elif type -P md5 > /dev/null; then [ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0 else @@ -159,7 +178,7 @@ check_file_signature() { if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then - vecho "Checking signature ..." + __vecho "Checking signature ..." if type -P gpg > /dev/null; then gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0 @@ -183,13 +202,25 @@ get_snapshot_timestamp() { sync_local() { local file="$1" - vecho "Syncing local tree ..." + __vecho "Syncing local tree ..." + + local ownership="portage:portage" + if has usersync ${FEATURES} ; then + case "${USERLAND}" in + BSD) + ownership=$(stat -f '%Su:%Sg' "${repo_location}") + ;; + *) + ownership=$(stat -c '%U:%G' "${repo_location}") + ;; + esac + fi if type -P tarsync > /dev/null ; then - local chown_opts="-o portage -g portage" - chown portage:portage portage > /dev/null 2>&1 || chown_opts="" + local chown_opts="-o ${ownership%:*} -g ${ownership#*:}" + chown ${ownership} "${repo_location}" > /dev/null 2>&1 || chown_opts="" if ! tarsync $(vvecho -v) -s 1 ${chown_opts} \ - -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then + -e /distfiles -e /packages -e /local "${file}" "${repo_location}"; then eecho "tarsync failed; tarball is corrupt? (${file})" return 1 fi @@ -201,27 +232,29 @@ sync_local() { fi # Free disk space - rm -f "${file}" + ${keep} || rm -f "${file}" - chown portage:portage portage > /dev/null 2>&1 && \ - chown -R portage:portage portage + local rsync_opts="${PORTAGE_RSYNC_OPTS} ${PORTAGE_RSYNC_EXTRA_OPTS}" + if chown ${ownership} portage > /dev/null 2>&1; then + chown -R ${ownership} portage + rsync_opts+=" --owner --group" + fi cd portage - rsync -av --progress --stats --delete --delete-after \ - --exclude='/distfiles' --exclude='/packages' \ - --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}" + rsync ${rsync_opts} . "${repo_location%%/}" cd .. - vecho "Cleaning up ..." + __vecho "Cleaning up ..." rm -fr portage fi if has metadata-transfer ${FEATURES} ; then - vecho "Updating cache ..." - emerge --metadata + __vecho "Updating cache ..." + "${PORTAGE_BIN_PATH}/emerge" --metadata fi - [ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync + local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync + [ -x "${post_sync}" ] && "${post_sync}" # --quiet suppresses output if there are no relevant news items - has news ${FEATURES} && emerge --check-news --quiet + has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet return 0 } @@ -251,14 +284,15 @@ do_snapshot() { for mirror in ${GENTOO_MIRRORS} ; do - vecho "Trying to retrieve ${date} snapshot from ${mirror} ..." + mirror=${mirror%/} + __vecho "Trying to retrieve ${date} snapshot from ${mirror} ..." for compression in ${compressions} ; do local file="portage-${date}.tar.${compression}" local digest="${file}.md5sum" local signature="${file}.gpgsig" - if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then + if [ -s "${DISTDIR}/${file}" -a -s "${DISTDIR}/${digest}" -a -s "${DISTDIR}/${signature}" ] ; then check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \ check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \ have_files=1 @@ -280,8 +314,8 @@ do_snapshot() { # if [ ${have_files} -eq 1 ]; then - vecho "Getting snapshot timestamp ..." - local snapshot_timestamp=$(get_snapshot_timestamp "${file}") + __vecho "Getting snapshot timestamp ..." + local snapshot_timestamp=$(get_snapshot_timestamp "${DISTDIR}/${file}") if [ ${ignore_timestamp} == 0 ]; then if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then @@ -310,7 +344,7 @@ do_snapshot() { # # Remove files and use a different mirror # - rm -f "${file}" "${digest}" "${signature}" + rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}" fi done @@ -318,12 +352,12 @@ do_snapshot() { done if [ ${have_files} -eq 1 ]; then - sync_local "${file}" && r=0 + sync_local "${DISTDIR}/${file}" && r=0 else - vecho "${date} snapshot was not found" + __vecho "${date} snapshot was not found" fi - - rm -f "${file}" "${digest}" "${signature}" + + ${keep} || rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}" return "${r}" } @@ -331,9 +365,9 @@ do_latest_snapshot() { local attempts=0 local r=1 - vecho "Fetching most recent snapshot ..." + __vecho "Fetching most recent snapshot ..." - # The snapshot for a given day is generated at 01:45 UTC on the following + # The snapshot for a given day is generated at 00:45 UTC on the following # day, so the current day's snapshot (going by UTC time) hasn't been # generated yet. Therefore, always start by looking for the previous day's # snapshot (for attempts=1, subtract 1 day from the current UTC time). @@ -349,10 +383,10 @@ do_latest_snapshot() { local start_time=$(get_utc_date_in_seconds) local start_hour=$(get_date_part ${start_time} "%H") - # Daily snapshots are created at 1:45 AM and are not - # available until after 2 AM. Don't waste time trying + # Daily snapshots are created at 00:45 and are not + # available until after 01:00. Don't waste time trying # to fetch a snapshot before it's been created. - if [ ${start_hour} -lt 2 ] ; then + if [ ${start_hour} -lt 1 ] ; then (( start_time -= 86400 )) fi local snapshot_date=$(get_date_part ${start_time} "%Y%m%d") @@ -361,8 +395,8 @@ do_latest_snapshot() { while (( ${attempts} < 40 )) ; do (( attempts++ )) (( snapshot_date_seconds -= 86400 )) - # snapshots are created at 1:45 AM - (( approx_snapshot_time = snapshot_date_seconds + 86400 + 6300 )) + # snapshots are created at 00:45 + (( approx_snapshot_time = snapshot_date_seconds + 86400 + 2700 )) (( timestamp_difference = existing_timestamp - approx_snapshot_time )) [ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference )) snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d") @@ -388,7 +422,7 @@ do_latest_snapshot() { "snapshot. In order to force sync," \ "use the --revert option or remove" \ "the timestamp file located at" \ - "'${PORTDIR}/metadata/timestamp.x'." | fmt -w 70 | \ + "'${repo_location}/metadata/timestamp.x'." | fmt -w 70 | \ while read -r line ; do ewarn "${line}" done @@ -408,9 +442,10 @@ do_latest_snapshot() { usage() { cat <<-EOF Usage: $0 [options] - + Options: --revert=yyyymmdd Revert to snapshot + -k, --keep Keep snapshots in DISTDIR (don't delete) -q, --quiet Only output errors -v, --verbose Enable verbose output -x, --debug Enable debug output @@ -427,14 +462,12 @@ usage() { main() { local arg local revert_date - - [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}" - cd "${DISTDIR}" for arg in "$@" ; do local v=${arg#*=} case ${arg} in -h|--help) usage ;; + -k|--keep) keep=true ;; -q|--quiet) PORTAGE_QUIET=1 ;; -v|--verbose) do_verbose=1 ;; -x|--debug) do_debug=1 ;; @@ -443,16 +476,39 @@ main() { esac done + [[ -d ${repo_location} ]] || mkdir -p "${repo_location}" + if [[ ! -w ${repo_location} ]] ; then + eecho "Repository '${repo_name}' is not writable: ${repo_location}" + exit 1 + fi + + [[ -d ${PORTAGE_TMPDIR}/portage ]] || mkdir -p "${PORTAGE_TMPDIR}/portage" + TMPDIR=$(mktemp -d "${PORTAGE_TMPDIR}/portage/webrsync-XXXXXX") + if [[ ! -w ${TMPDIR} ]] ; then + eecho "TMPDIR is not writable: ${TMPDIR}" + exit 1 + fi + trap 'cd / ; rm -rf "${TMPDIR}"' EXIT + cd "${TMPDIR}" || exit 1 + + ${keep} || DISTDIR=${TMPDIR} + [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}" + + if ${keep} && [[ ! -w ${DISTDIR} ]] ; then + eecho "DISTDIR is not writable: ${DISTDIR}" + exit 1 + fi + # This is a sanity check to help prevent people like funtoo users # from accidentally wiping out their git tree. - if [[ -n $SYNC && ${SYNC#rsync:} = $SYNC ]] ; then - echo "The current SYNC variable setting does not refer to an rsync URI:" >&2 + if [[ -n ${repo_sync_type} && ${repo_sync_type} != rsync ]] ; then + echo "The current sync-type attribute of repository 'gentoo' is not set to 'rsync':" >&2 echo >&2 - echo " SYNC=$SYNC" >&2 + echo " sync-type=${repo_sync_type}" >&2 echo >&2 echo "If you intend to use emerge-webrsync then please" >&2 - echo "adjust SYNC to refer to an rsync URI." >&2 - echo "emerge-webrsync exiting due to abnormal SYNC setting." >&2 + echo "adjust sync-type and sync-uri attributes to refer to rsync." >&2 + echo "emerge-webrsync exiting due to abnormal sync-type setting." >&2 exit 1 fi diff --git a/portage_with_autodep/bin/env-update b/portage_with_autodep/bin/env-update index 8a69f2b..b500c54 100755 --- a/portage_with_autodep/bin/env-update +++ b/portage_with_autodep/bin/env-update @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -25,12 +25,12 @@ if len(sys.argv) > 1: print("!!! Invalid command line options!\n") usage(1) -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True + try: portage.env_update(makelinks) except IOError as e: diff --git a/portage_with_autodep/bin/etc-update b/portage_with_autodep/bin/etc-update index 1edc91f..1a99231 100755 --- a/portage_with_autodep/bin/etc-update +++ b/portage_with_autodep/bin/etc-update @@ -62,7 +62,7 @@ do_mv_ln() { } scan() { - echo "Scanning Configuration files..." + ${QUIET} || echo "Scanning Configuration files..." rm -rf "${TMP}"/files > /dev/null 2>&1 mkdir "${TMP}"/files || die "Failed mkdir command!" count=0 @@ -107,13 +107,13 @@ scan() { for mpath in ${CONFIG_PROTECT_MASK}; do mpath="${EROOT%/}${mpath}" if [[ "${rpath}" == "${mpath}"* ]] ; then - echo "Updating masked file: ${live_file}" + ${QUIET} || echo "Updating masked file: ${live_file}" mv "${cfg_file}" "${live_file}" continue 2 fi done if [[ ! -f ${file} ]] ; then - echo "Skipping non-file ${file} ..." + ${QUIET} || echo "Skipping non-file ${file} ..." continue fi @@ -140,7 +140,7 @@ scan() { fi if [[ ${MATCHES} == 1 ]] ; then - echo "Automerging trivial changes in: ${live_file}" + ${QUIET} || echo "Automerging trivial changes in: ${live_file}" do_mv_ln "${cfg_file}" "${live_file}" continue else @@ -190,6 +190,7 @@ parse_automode_flag() { parse_automode_flag -3 export mv_opts=" ${mv_opts} " mv_opts="${mv_opts// -i / }" + NONINTERACTIVE_MV=true ;; -3) input=0 @@ -547,9 +548,9 @@ die() { local msg=$1 exitcode=${2:-1} if [ ${exitcode} -eq 0 ] ; then - printf 'Exiting: %b\n' "${msg}" + ${QUIET} || printf 'Exiting: %b\n' "${msg}" scan > /dev/null - [ ${count} -gt 0 ] && echo "NOTE: ${count} updates remaining" + ! ${QUIET} && [ ${count} -gt 0 ] && echo "NOTE: ${count} updates remaining" else error "${msg}" fi @@ -574,6 +575,7 @@ usage() { -d, --debug Enable shell debugging -h, --help Show help and run away -p, --preen Automerge trivial changes only and quit + -q, --quiet Show only essential output -v, --verbose Show settings and such along the way -V, --version Show version and trundle away @@ -599,12 +601,15 @@ declare title="Gentoo's etc-update tool!" PREEN=false SET_X=false +QUIET=false VERBOSE=false +NONINTERACTIVE_MV=false while [[ -n $1 ]] ; do case $1 in -d|--debug) SET_X=true;; -h|--help) usage;; -p|--preen) PREEN=true;; + -q|--quiet) QUIET=true;; -v|--verbose) VERBOSE=true;; -V|--version) emerge --version; exit 0;; --automode) parse_automode_flag $2 && shift || usage 1 "Invalid mode '$2'";; @@ -615,7 +620,7 @@ while [[ -n $1 ]] ; do done ${SET_X} && set -x -type portageq >/dev/null || die "missing portageq" +type -P portageq >/dev/null || die "missing portageq" portage_vars=( CONFIG_PROTECT{,_MASK} PORTAGE_CONFIGROOT @@ -625,7 +630,7 @@ portage_vars=( USERLAND NOCOLOR ) -eval $(portageq envvar -v ${portage_vars[@]}) +eval $(${PORTAGE_PYTHON:+"${PORTAGE_PYTHON}"} "$(type -P portageq)" envvar -v ${portage_vars[@]}) export PORTAGE_TMPDIR SCAN_PATHS=${*:-${CONFIG_PROTECT}} @@ -692,6 +697,11 @@ else fi fi +if ${NONINTERACTIVE_MV} ; then + export mv_opts=" ${mv_opts} " + mv_opts="${mv_opts// -i / }" +fi + if ${VERBOSE} ; then for v in ${portage_vars[@]} ${cfg_vars[@]} TMP SCAN_PATHS ; do echo "${v}=${!v}" diff --git a/portage_with_autodep/bin/filter-bash-environment.py b/portage_with_autodep/bin/filter-bash-environment.py index b9aec96..3d4b3ec 100755 --- a/portage_with_autodep/bin/filter-bash-environment.py +++ b/portage_with_autodep/bin/filter-bash-environment.py @@ -1,10 +1,9 @@ #!/usr/bin/python -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import codecs import io -import optparse import os import re import sys @@ -126,10 +125,19 @@ if __name__ == "__main__": "intact. The PATTERN is a space separated list of variable names" + \ " and it supports python regular expression syntax." usage = "usage: %s PATTERN" % os.path.basename(sys.argv[0]) - parser = optparse.OptionParser(description=description, usage=usage) - options, args = parser.parse_args(sys.argv[1:]) + args = sys.argv[1:] + + if '-h' in args or '--help' in args: + sys.stdout.write(usage + "\n") + sys.stdout.flush() + sys.exit(os.EX_OK) + if len(args) != 1: - parser.error("Missing required PATTERN argument.") + sys.stderr.write(usage + "\n") + sys.stderr.write("Exactly one PATTERN argument required.\n") + sys.stderr.flush() + sys.exit(2) + file_in = sys.stdin file_out = sys.stdout if sys.hexversion >= 0x3000000: diff --git a/portage_with_autodep/bin/fixpackages b/portage_with_autodep/bin/fixpackages index dc43ed2..e29d6ee 100755 --- a/portage_with_autodep/bin/fixpackages +++ b/portage_with_autodep/bin/fixpackages @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -7,21 +7,27 @@ from __future__ import print_function import os import sys -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os from portage.output import EOutput +from portage.util._argparse import ArgumentParser from textwrap import wrap from portage._global_updates import _global_updates mysettings = portage.settings mytrees = portage.db mtimedb = portage.mtimedb +description = """The fixpackages program performs package move updates on + configuration files, installed packages, and binary packages.""" +description = " ".join(description.split()) + +parser = ArgumentParser(description=description) +parser.parse_args() + if mysettings['ROOT'] != "/": out = EOutput() msg = "The fixpackages program is not intended for use with " + \ diff --git a/portage_with_autodep/bin/glsa-check b/portage_with_autodep/bin/glsa-check index a840c32..7fa3688 100755 --- a/portage_with_autodep/bin/glsa-check +++ b/portage_with_autodep/bin/glsa-check @@ -1,81 +1,79 @@ #!/usr/bin/python -# Copyright 2008-2011 Gentoo Foundation +# Copyright 2008-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import sys +import codecs -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os -from portage.output import * - -from optparse import OptionGroup, OptionParser +from portage.output import green, red, nocolor, white +from portage.util._argparse import ArgumentParser __program__ = "glsa-check" __author__ = "Marius Mauch <genone@gentoo.org>" __version__ = "1.0" -def cb_version(*args, **kwargs): - """Callback for --version""" - sys.stderr.write("\n"+ __program__ + ", version " + __version__ + "\n") - sys.stderr.write("Author: " + __author__ + "\n") - sys.stderr.write("This program is licensed under the GPL, version 2\n\n") - sys.exit(0) - # option parsing -parser = OptionParser(usage="%prog <option> [glsa-list]", - version="%prog "+ __version__) -parser.epilog = "glsa-list can contain an arbitrary number of GLSA ids," \ +epilog = "glsa-list can contain an arbitrary number of GLSA ids," \ " filenames containing GLSAs or the special identifiers" \ " 'all', 'new' and 'affected'" +parser = ArgumentParser(usage=__program__ + " <option> [glsa-list]", + epilog=epilog) -modes = OptionGroup(parser, "Modes") -modes.add_option("-l", "--list", action="store_const", +modes = parser.add_argument_group("Modes") +modes.add_argument("-l", "--list", action="store_const", const="list", dest="mode", help="List all unapplied GLSA") -modes.add_option("-d", "--dump", action="store_const", +modes.add_argument("-d", "--dump", action="store_const", const="dump", dest="mode", help="Show all information about the given GLSA") -modes.add_option("", "--print", action="store_const", +modes.add_argument("--print", action="store_const", const="dump", dest="mode", help="Alias for --dump") -modes.add_option("-t", "--test", action="store_const", +modes.add_argument("-t", "--test", action="store_const", const="test", dest="mode", help="Test if this system is affected by the given GLSA") -modes.add_option("-p", "--pretend", action="store_const", +modes.add_argument("-p", "--pretend", action="store_const", const="pretend", dest="mode", help="Show the necessary commands to apply this GLSA") -modes.add_option("-f", "--fix", action="store_const", +modes.add_argument("-f", "--fix", action="store_const", const="fix", dest="mode", help="Try to auto-apply this GLSA (experimental)") -modes.add_option("-i", "--inject", action="store_const", dest="mode", - help="Inject the given GLSA into the checkfile") -modes.add_option("-m", "--mail", action="store_const", +modes.add_argument("-i", "--inject", action="store_const", + const="inject", dest="mode", + help="inject the given GLSA into the glsa_injected file") +modes.add_argument("-m", "--mail", action="store_const", const="mail", dest="mode", help="Send a mail with the given GLSAs to the administrator") -parser.add_option_group(modes) -parser.remove_option("--version") -parser.add_option("-V", "--version", action="callback", - callback=cb_version, help="Some information about this tool") -parser.add_option("-v", "--verbose", action="store_true", dest="verbose", +parser.add_argument("-V", "--version", action="store_true", + help="Some information about this tool") +parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", help="Print more information") -parser.add_option("-n", "--nocolor", action="callback", - callback=lambda *args, **kwargs: nocolor(), +parser.add_argument("-n", "--nocolor", action="store_true", help="Disable colors") -parser.add_option("-e", "--emergelike", action="store_false", dest="least_change", +parser.add_argument("-e", "--emergelike", action="store_false", dest="least_change", help="Do not use a least-change algorithm") -parser.add_option("-c", "--cve", action="store_true", dest="list_cve", +parser.add_argument("-c", "--cve", action="store_true", dest="list_cve", help="Show CAN ids in listing mode") -options, params = parser.parse_args() +options, params = parser.parse_known_args() + +if options.nocolor: + nocolor() + +if options.version: + sys.stderr.write("\n"+ __program__ + ", version " + __version__ + "\n") + sys.stderr.write("Author: " + __author__ + "\n") + sys.stderr.write("This program is licensed under the GPL, version 2\n\n") + sys.exit(0) mode = options.mode least_change = options.least_change @@ -101,7 +99,8 @@ elif mode == "list" and not params: params.append("new") # delay this for speed increase -from portage.glsa import * +from portage.glsa import (Glsa, GlsaTypeException, GlsaFormatException, + get_applied_glsas, get_glsa_list) eroot = portage.settings['EROOT'] vardb = portage.db[eroot]["vartree"].dbapi @@ -117,7 +116,7 @@ glsalist = [] if "new" in params: glsalist = todolist params.remove("new") - + if "all" in params: glsalist = completelist params.remove("all") @@ -142,8 +141,17 @@ for p in params[:]: glsalist.extend([g for g in params if g not in glsalist]) -def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr): - fd2.write(white("[A]")+" means this GLSA was already applied,\n") +def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr, encoding="utf-8"): + # Get to the raw streams in py3k before wrapping them with an encoded writer + # to avoid writing bytes to a text stream (stdout/stderr are text streams + # by default in py3k) + if hasattr(fd1, "buffer"): + fd1 = fd1.buffer + if hasattr(fd2, "buffer"): + fd2 = fd2.buffer + fd1 = codecs.getwriter(encoding)(fd1) + fd2 = codecs.getwriter(encoding)(fd2) + fd2.write(white("[A]")+" means this GLSA was marked as applied (injected),\n") fd2.write(green("[U]")+" means the system is not affected and\n") fd2.write(red("[N]")+" indicates that the system might be affected.\n\n") @@ -155,7 +163,7 @@ def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr): if verbose: fd2.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue - if myglsa.isApplied(): + if myglsa.isInjected(): status = "[A]" color = white elif myglsa.isVulnerable(): @@ -186,7 +194,7 @@ def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr): fd1.write(")") if list_cve: fd1.write(" "+(",".join([r[:13] for r in myglsa.references if r[:4] in ["CAN-", "CVE-"]]))) - fd1.write("\n") + fd1.write("\n") return 0 if mode == "list": @@ -204,39 +212,46 @@ if mode in ["dump", "fix", "inject", "pretend"]: if mode == "dump": myglsa.dump() elif mode == "fix": - sys.stdout.write("fixing "+myid+"\n") - mergelist = myglsa.getMergeList(least_change=least_change) - for pkg in mergelist: - sys.stdout.write(">>> merging "+pkg+"\n") - # using emerge for the actual merging as it contains the dependency - # code and we want to be consistent in behaviour. Also this functionality - # will be integrated in emerge later, so it shouldn't hurt much. - emergecmd = "emerge --oneshot " + portage.settings["EMERGE_OPTS"] + " =" + pkg - if verbose: - sys.stderr.write(emergecmd+"\n") - exitcode = os.system(emergecmd) - # system() returns the exitcode in the high byte of a 16bit integer - if exitcode >= 1<<8: - exitcode >>= 8 - if exitcode: - sys.exit(exitcode) - myglsa.inject() + sys.stdout.write("Fixing GLSA "+myid+"\n") + if not myglsa.isVulnerable(): + sys.stdout.write(">>> no vulnerable packages installed\n") + else: + mergelist = myglsa.getMergeList(least_change=least_change) + if mergelist == []: + sys.stdout.write(">>> cannot fix GLSA, no unaffected packages available\n") + sys.exit(2) + for pkg in mergelist: + sys.stdout.write(">>> merging "+pkg+"\n") + # using emerge for the actual merging as it contains the dependency + # code and we want to be consistent in behaviour. Also this functionality + # will be integrated in emerge later, so it shouldn't hurt much. + emergecmd = "emerge --oneshot " + " =" + pkg + if verbose: + sys.stderr.write(emergecmd+"\n") + exitcode = os.system(emergecmd) + # system() returns the exitcode in the high byte of a 16bit integer + if exitcode >= 1<<8: + exitcode >>= 8 + if exitcode: + sys.exit(exitcode) + if len(mergelist): + sys.stdout.write("\n") elif mode == "pretend": sys.stdout.write("Checking GLSA "+myid+"\n") - mergelist = myglsa.getMergeList(least_change=least_change) - if mergelist: - sys.stdout.write("The following updates will be performed for this GLSA:\n") - for pkg in mergelist: - oldver = None - for x in vardb.match(portage.cpv_getkey(pkg)): - if vardb.aux_get(x, ["SLOT"]) == portdb.aux_get(pkg, ["SLOT"]): - oldver = x - if oldver == None: - raise ValueError("could not find old version for package %s" % pkg) - oldver = oldver[len(portage.cpv_getkey(oldver))+1:] - sys.stdout.write(" " + pkg + " (" + oldver + ")\n") + if not myglsa.isVulnerable(): + sys.stdout.write(">>> no vulnerable packages installed\n") else: - sys.stdout.write("Nothing to do for this GLSA\n") + mergedict = {} + for (vuln, update) in myglsa.getAffectionTable(least_change=least_change): + mergedict.setdefault(update, []).append(vuln) + + sys.stdout.write(">>> The following updates will be performed for this GLSA:\n") + for pkg in mergedict: + if pkg != "": + sys.stdout.write(" " + pkg + " (vulnerable: " + ", ".join(mergedict[pkg]) + ")\n") + if "" in mergedict: + sys.stdout.write("\n>>> For the following packages, no upgrade path exists:\n") + sys.stdout.write(" " + ", ".join(mergedict[""])) elif mode == "inject": sys.stdout.write("injecting " + myid + "\n") myglsa.inject() @@ -268,9 +283,9 @@ if mode == "test": # mail mode as requested by solar if mode == "mail": import portage.mail, socket - from io import StringIO + from io import BytesIO from email.mime.text import MIMEText - + # color doesn't make any sense for mail nocolor() @@ -278,7 +293,7 @@ if mode == "mail": myrecipient = portage.settings["PORTAGE_ELOG_MAILURI"].split()[0] else: myrecipient = "root@localhost" - + if "PORTAGE_ELOG_MAILFROM" in portage.settings: myfrom = portage.settings["PORTAGE_ELOG_MAILFROM"] else: @@ -287,11 +302,13 @@ if mode == "mail": mysubject = "[glsa-check] Summary for %s" % socket.getfqdn() # need a file object for summarylist() - myfd = StringIO() - myfd.write("GLSA Summary report for host %s\n" % socket.getfqdn()) - myfd.write("(Command was: %s)\n\n" % " ".join(sys.argv)) + myfd = BytesIO() + line = "GLSA Summary report for host %s\n" % socket.getfqdn() + myfd.write(line.encode("utf-8")) + line = "(Command was: %s)\n\n" % " ".join(sys.argv) + myfd.write(line.encode("utf-8")) summarylist(glsalist, fd1=myfd, fd2=myfd) - summary = str(myfd.getvalue()) + summary = myfd.getvalue().decode("utf-8") myfd.close() myattachments = [] @@ -302,16 +319,17 @@ if mode == "mail": if verbose: sys.stderr.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue - myfd = StringIO() + myfd = BytesIO() myglsa.dump(outstream=myfd) - myattachments.append(MIMEText(str(myfd.getvalue()), _charset="utf8")) + attachment = myfd.getvalue().decode("utf-8") + myattachments.append(MIMEText(attachment, _charset="utf8")) myfd.close() - + mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, summary, myattachments) portage.mail.send_mail(portage.settings, mymessage) - + sys.exit(0) - + # something wrong here, all valid paths are covered with sys.exit() sys.stderr.write("nothing more to do\n") sys.exit(2) diff --git a/portage_with_autodep/bin/helper-functions.sh b/portage_with_autodep/bin/helper-functions.sh index afe14af..b9bc74a 100755 --- a/portage_with_autodep/bin/helper-functions.sh +++ b/portage_with_autodep/bin/helper-functions.sh @@ -10,42 +10,45 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh # # API functions for doing parallel processing # -numjobs() { +makeopts_jobs() { # Copied from eutils.eclass:makeopts_jobs() local jobs=$(echo " ${MAKEOPTS} " | \ sed -r -n 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p') echo ${jobs:-1} } -multijob_init() { +__multijob_init() { # Setup a pipe for children to write their pids to when they finish. - mj_control_pipe=$(mktemp -t multijob.XXXXXX) - rm "${mj_control_pipe}" - mkfifo "${mj_control_pipe}" - exec {mj_control_fd}<>${mj_control_pipe} - rm -f "${mj_control_pipe}" + # We have to allocate two fd's because POSIX has undefined behavior + # when you open a FIFO for simultaneous read/write. #487056 + local pipe=$(mktemp -t multijob.XXXXXX) + rm -f "${pipe}" + mkfifo -m 600 "${pipe}" + __redirect_alloc_fd mj_write_fd "${pipe}" + __redirect_alloc_fd mj_read_fd "${pipe}" + rm -f "${pipe}" # See how many children we can fork based on the user's settings. - mj_max_jobs=$(numjobs) + mj_max_jobs=$(makeopts_jobs "$@") mj_num_jobs=0 } -multijob_child_init() { - trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT +__multijob_child_init() { + trap 'echo ${BASHPID:-$(__bashpid)} $? >&'${mj_write_fd} EXIT trap 'exit 1' INT TERM } -multijob_finish_one() { +__multijob_finish_one() { local pid ret - read -r -u ${mj_control_fd} pid ret + read -r -u ${mj_read_fd} pid ret : $(( --mj_num_jobs )) return ${ret} } -multijob_finish() { +__multijob_finish() { local ret=0 while [[ ${mj_num_jobs} -gt 0 ]] ; do - multijob_finish_one + __multijob_finish_one : $(( ret |= $? )) done # Let bash clean up its internal child tracking state. @@ -53,10 +56,42 @@ multijob_finish() { return ${ret} } -multijob_post_fork() { +__multijob_post_fork() { : $(( ++mj_num_jobs )) if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then - multijob_finish_one + __multijob_finish_one fi return $? } + +# @FUNCTION: __redirect_alloc_fd +# @USAGE: <var> <file> [redirection] +# @DESCRIPTION: +# Find a free fd and redirect the specified file via it. Store the new +# fd in the specified variable. Useful for the cases where we don't care +# about the exact fd #. +__redirect_alloc_fd() { + local var=$1 file=$2 redir=${3:-"<>"} + + if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then + # Newer bash provides this functionality. + eval "exec {${var}}${redir}'${file}'" + else + # Need to provide the functionality ourselves. + local fd=10 + local fddir=/dev/fd + # Prefer /proc/self/fd if available (/dev/fd + # doesn't work on solaris, see bug #474536). + [[ -d /proc/self/fd ]] && fddir=/proc/self/fd + while :; do + # Make sure the fd isn't open. It could be a char device, + # or a symlink (possibly broken) to something else. + if [[ ! -e ${fddir}/${fd} ]] && [[ ! -L ${fddir}/${fd} ]] ; then + eval "exec ${fd}${redir}'${file}'" && break + fi + [[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?' + : $(( ++fd )) + done + : $(( ${var} = fd )) + fi +} diff --git a/portage_with_autodep/bin/isolated-functions.sh b/portage_with_autodep/bin/isolated-functions.sh index dbf988b..b99aec7 100755 --- a/portage_with_autodep/bin/isolated-functions.sh +++ b/portage_with_autodep/bin/isolated-functions.sh @@ -1,7 +1,9 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}/eapi.sh" + # We need this next line for "die" and "assert". It expands # It _must_ preceed all the calls to die and assert. shopt -s expand_aliases @@ -15,7 +17,7 @@ assert() { done } -assert_sigpipe_ok() { +__assert_sigpipe_ok() { # When extracting a tar file like this: # # bzip2 -dc foo.tar.bz2 | tar xof - @@ -43,21 +45,21 @@ assert_sigpipe_ok() { shopt -s extdebug -# dump_trace([number of funcs on stack to skip], +# __dump_trace([number of funcs on stack to skip], # [whitespacing for filenames], # [whitespacing for line numbers]) -dump_trace() { +__dump_trace() { local funcname="" sourcefile="" lineno="" s="yes" n p declare -i strip=${1:-1} local filespacing=$2 linespacing=$3 - # The qa_call() function and anything before it are portage internals + # The __qa_call() function and anything before it are portage internals # that the user will not be interested in. Therefore, the stack trace - # should only show calls that come after qa_call(). + # should only show calls that come after __qa_call(). (( n = ${#FUNCNAME[@]} - 1 )) (( p = ${#BASH_ARGV[@]} )) while (( n > 0 )) ; do - [ "${FUNCNAME[${n}]}" == "qa_call" ] && break + [ "${FUNCNAME[${n}]}" == "__qa_call" ] && break (( p -= ${BASH_ARGC[${n}]} )) (( n-- )) done @@ -86,7 +88,7 @@ dump_trace() { } nonfatal() { - if has "${EAPI:-0}" 0 1 2 3 3_pre2 ; then + if ! ___eapi_has_nonfatal; then die "$FUNCNAME() not supported in this EAPI" fi if [[ $# -lt 1 ]]; then @@ -96,18 +98,24 @@ nonfatal() { PORTAGE_NONFATAL=1 "$@" } -helpers_die() { - case "${EAPI:-0}" in - 0|1|2|3) - echo -e "$@" >&2 - ;; - *) - die "$@" - ;; - esac +__bashpid() { + # The BASHPID variable is new to bash-4.0, so add a hack for older + # versions. This must be used like so: + # ${BASHPID:-$(__bashpid)} + sh -c 'echo ${PPID}' +} + +__helpers_die() { + if ___eapi_helpers_can_die; then + die "$@" + else + echo -e "$@" >&2 + fi } die() { + local IFS=$' \t\n' + if [[ $PORTAGE_NONFATAL -eq 1 ]]; then echo -e " $WARN*$NORMAL ${FUNCNAME[1]}: WARNING: $@" >&2 return 1 @@ -124,7 +132,7 @@ die() { # setup spacing to make output easier to read (( n = ${#FUNCNAME[@]} - 1 )) while (( n > 0 )) ; do - [ "${FUNCNAME[${n}]}" == "qa_call" ] && break + [ "${FUNCNAME[${n}]}" == "__qa_call" ] && break (( n-- )) done (( n == 0 )) && (( n = ${#FUNCNAME[@]} - 1 )) @@ -140,14 +148,14 @@ die() { # get a stack trace, so at least report the phase that failed. local phase_str= [[ -n $EBUILD_PHASE ]] && phase_str=" ($EBUILD_PHASE phase)" - eerror "ERROR: $CATEGORY/$PF failed${phase_str}:" + eerror "ERROR: ${CATEGORY}/${PF}::${PORTAGE_REPO_NAME} failed${phase_str}:" eerror " ${*:-(no error message)}" eerror - # dump_trace is useless when the main script is a helper binary + # __dump_trace is useless when the main script is a helper binary local main_index (( main_index = ${#BASH_SOURCE[@]} - 1 )) if has ${BASH_SOURCE[$main_index]##*/} ebuild.sh misc-functions.sh ; then - dump_trace 2 ${filespacing} ${linespacing} + __dump_trace 2 ${filespacing} ${linespacing} eerror " $(printf "%${filespacing}s" "${BASH_SOURCE[1]##*/}"), line $(printf "%${linespacing}s" "${BASH_LINENO[0]}"): Called die" eerror "The specific snippet of code:" # This scans the file that called die and prints out the logic that @@ -173,39 +181,12 @@ die() { | while read -r n ; do eerror " ${n#RETAIN-LEADING-SPACE}" ; done eerror fi - eerror "If you need support, post the output of \`emerge --info '=$CATEGORY/$PF'\`," - eerror "the complete build log and the output of \`emerge -pqv '=$CATEGORY/$PF'\`." - if [[ -n ${EBUILD_OVERLAY_ECLASSES} ]] ; then - eerror "This ebuild used the following eclasses from overlays:" - local x - for x in ${EBUILD_OVERLAY_ECLASSES} ; do - eerror " ${x}" - done - fi - if [ "${EMERGE_FROM}" != "binary" ] && \ - ! has ${EBUILD_PHASE} prerm postrm && \ - [ "${EBUILD#${PORTDIR}/}" == "${EBUILD}" ] ; then - local overlay=${EBUILD%/*} - overlay=${overlay%/*} - overlay=${overlay%/*} - if [[ -n $PORTAGE_REPO_NAME ]] ; then - eerror "This ebuild is from an overlay named" \ - "'$PORTAGE_REPO_NAME': '${overlay}/'" - else - eerror "This ebuild is from an overlay: '${overlay}/'" - fi - elif [[ -n $PORTAGE_REPO_NAME && -f "$PORTDIR"/profiles/repo_name ]] ; then - local portdir_repo_name=$(<"$PORTDIR"/profiles/repo_name) - if [[ -n $portdir_repo_name && \ - $portdir_repo_name != $PORTAGE_REPO_NAME ]] ; then - eerror "This ebuild is from a repository" \ - "named '$PORTAGE_REPO_NAME'" - fi - fi + eerror "If you need support, post the output of \`emerge --info '=${CATEGORY}/${PF}::${PORTAGE_REPO_NAME}'\`," + eerror "the complete build log and the output of \`emerge -pqv '=${CATEGORY}/${PF}::${PORTAGE_REPO_NAME}'\`." # Only call die hooks here if we are executed via ebuild.sh or # misc-functions.sh, since those are the only cases where the environment - # contains the hook functions. When necessary (like for helpers_die), die + # contains the hook functions. When necessary (like for __helpers_die), die # hooks are automatically called later by a misc-functions.sh invocation. if has ${BASH_SOURCE[$main_index]##*/} ebuild.sh misc-functions.sh && \ [[ ${EBUILD_PHASE} != depend ]] ; then @@ -218,7 +199,8 @@ die() { if [[ -n ${PORTAGE_LOG_FILE} ]] ; then eerror "The complete build log is located at '${PORTAGE_LOG_FILE}'." - if [[ ${PORTAGE_LOG_FILE} != ${T}/* ]] ; then + if [[ ${PORTAGE_LOG_FILE} != ${T}/* ]] && \ + ! has fail-clean ${FEATURES} ; then # Display path to symlink in ${T}, as requested in bug #412865. local log_ext=log [[ ${PORTAGE_LOG_FILE} != *.log ]] && log_ext+=.${PORTAGE_LOG_FILE##*.} @@ -241,26 +223,20 @@ die() { [[ -n $PORTAGE_IPC_DAEMON ]] && "$PORTAGE_BIN_PATH"/ebuild-ipc exit 1 # subshell die support - [[ $BASHPID = $EBUILD_MASTER_PID ]] || kill -s SIGTERM $EBUILD_MASTER_PID + [[ ${BASHPID:-$(__bashpid)} == ${EBUILD_MASTER_PID} ]] || kill -s SIGTERM ${EBUILD_MASTER_PID} exit 1 } -# We need to implement diefunc() since environment.bz2 files contain -# calls to it (due to alias expansion). -diefunc() { - die "${@}" -} - -quiet_mode() { +__quiet_mode() { [[ ${PORTAGE_QUIET} -eq 1 ]] } -vecho() { - quiet_mode || echo "$@" +__vecho() { + __quiet_mode || echo "$@" } # Internal logging function, don't use this in ebuilds -elog_base() { +__elog_base() { local messagetype [ -z "${1}" -o -z "${T}" -o ! -d "${T}/logging" ] && return 1 case "${1}" in @@ -269,7 +245,7 @@ elog_base() { shift ;; *) - vecho -e " ${BAD}*${NORMAL} Invalid use of internal function elog_base(), next message will not be logged" + __vecho -e " ${BAD}*${NORMAL} Invalid use of internal function __elog_base(), next message will not be logged" return 1 ;; esac @@ -281,17 +257,17 @@ elog_base() { } eqawarn() { - elog_base QA "$*" + __elog_base QA "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -e "$@" | while read -r ; do - vecho " $WARN*$NORMAL $REPLY" >&2 + __vecho " $WARN*$NORMAL $REPLY" >&2 done LAST_E_CMD="eqawarn" return 0 } elog() { - elog_base LOG "$*" + __elog_base LOG "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -e "$@" | while read -r ; do echo " $GOOD*$NORMAL $REPLY" @@ -300,26 +276,8 @@ elog() { return 0 } -esyslog() { - local pri= - local tag= - - if [ -x /usr/bin/logger ] - then - pri="$1" - tag="$2" - - shift 2 - [ -z "$*" ] && return 0 - - /usr/bin/logger -p "${pri}" -t "${tag}" -- "$*" - fi - - return 0 -} - einfo() { - elog_base INFO "$*" + __elog_base INFO "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -e "$@" | while read -r ; do echo " $GOOD*$NORMAL $REPLY" @@ -329,7 +287,7 @@ einfo() { } einfon() { - elog_base INFO "$*" + __elog_base INFO "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -ne " ${GOOD}*${NORMAL} $*" LAST_E_CMD="einfon" @@ -337,7 +295,7 @@ einfon() { } ewarn() { - elog_base WARN "$*" + __elog_base WARN "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -e "$@" | while read -r ; do echo " $WARN*$NORMAL $RC_INDENTATION$REPLY" >&2 @@ -347,7 +305,7 @@ ewarn() { } eerror() { - elog_base ERROR "$*" + __elog_base ERROR "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo echo -e "$@" | while read -r ; do echo " $BAD*$NORMAL $RC_INDENTATION$REPLY" >&2 @@ -372,7 +330,7 @@ ebegin() { return 0 } -_eend() { +__eend() { local retval=${1:-0} efunc=${2:-eerror} msg shift 2 @@ -399,13 +357,13 @@ eend() { local retval=${1:-0} shift - _eend ${retval} eerror "$*" + __eend ${retval} eerror "$*" LAST_E_CMD="eend" return ${retval} } -unset_colors() { +__unset_colors() { COLS=80 ENDCOL= @@ -417,7 +375,7 @@ unset_colors() { BRACKET= } -set_colors() { +__set_colors() { COLS=${COLUMNS:-0} # bash's internal COLUMNS variable # Avoid wasteful stty calls during the "depend" phases. # If stdout is a pipe, the parent process can export COLUMNS @@ -450,10 +408,10 @@ RC_DOT_PATTERN='' case "${NOCOLOR:-false}" in yes|true) - unset_colors + __unset_colors ;; no|false) - set_colors + __set_colors ;; esac @@ -504,4 +462,24 @@ has() { return 1 } +__repo_key() { + local appropriate_section=0 exit_status=1 line saved_extglob_shopt=$(shopt -p extglob) + shopt -s extglob + while read line; do + [[ ${appropriate_section} == 0 && ${line} == "[$1]" ]] && appropriate_section=1 && continue + [[ ${appropriate_section} == 1 && ${line} == "["*"]" ]] && appropriate_section=0 && continue + # If a conditional expression like [[ ${line} == $2*( )=* ]] is used + # then bash-3.2 produces an error like the following when the file is + # sourced: syntax error in conditional expression: unexpected token `(' + # Therefore, use a regular expression for compatibility. + if [[ ${appropriate_section} == 1 && ${line} =~ ^${2}[[:space:]]*= ]]; then + echo "${line##$2*( )=*( )}" + exit_status=0 + break + fi + done <<< "${PORTAGE_REPOSITORIES}" + eval "${saved_extglob_shopt}" + return ${exit_status} +} + true diff --git a/portage_with_autodep/bin/lock-helper.py b/portage_with_autodep/bin/lock-helper.py index dfb8876..128e4dd 100755 --- a/portage_with_autodep/bin/lock-helper.py +++ b/portage_with_autodep/bin/lock-helper.py @@ -1,11 +1,12 @@ #!/usr/bin/python -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os import sys sys.path.insert(0, os.environ['PORTAGE_PYM_PATH']) import portage +portage._internal_caller = True portage._disable_legacy_globals() def main(args): diff --git a/portage_with_autodep/bin/misc-functions.sh b/portage_with_autodep/bin/misc-functions.sh index 564af85..5ccf7c2 100755 --- a/portage_with_autodep/bin/misc-functions.sh +++ b/portage_with_autodep/bin/misc-functions.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # Miscellaneous shell functions that make use of the ebuild env but don't need @@ -17,8 +17,9 @@ shift $# source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}/ebuild.sh" install_symlink_html_docs() { - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi cd "${ED}" || die "cd failed" #symlink the html documentation (if DOC_SYMLINKS_DIR is set in make.conf) if [ -n "${DOC_SYMLINKS_DIR}" ] ; then @@ -30,10 +31,10 @@ install_symlink_html_docs() { done if [ -n "${mydocdir}" ] ; then local mysympath - if [ -z "${SLOT}" -o "${SLOT}" = "0" ] ; then + if [ -z "${SLOT}" -o "${SLOT%/*}" = "0" ] ; then mysympath="${DOC_SYMLINKS_DIR}/${CATEGORY}/${PN}" else - mysympath="${DOC_SYMLINKS_DIR}/${CATEGORY}/${PN}-${SLOT}" + mysympath="${DOC_SYMLINKS_DIR}/${CATEGORY}/${PN}-${SLOT%/*}" fi einfo "Symlinking ${mysympath} to the HTML documentation" dodir "${DOC_SYMLINKS_DIR}/${CATEGORY}" @@ -43,7 +44,20 @@ install_symlink_html_docs() { } # replacement for "readlink -f" or "realpath" +READLINK_F_WORKS="" canonicalize() { + if [[ -z ${READLINK_F_WORKS} ]] ; then + if [[ $(readlink -f -- /../ 2>/dev/null) == "/" ]] ; then + READLINK_F_WORKS=true + else + READLINK_F_WORKS=false + fi + fi + if ${READLINK_F_WORKS} ; then + readlink -f -- "$@" + return + fi + local f=$1 b n=10 wd=$(pwd) while (( n-- > 0 )); do while [[ ${f: -1} = / && ${#f} -gt 1 ]]; do @@ -66,8 +80,9 @@ canonicalize() { prepcompress() { local -a include exclude incl_d incl_f local f g i real_f real_d - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi # Canonicalize path names and check for their existence. real_d=$(canonicalize "${ED}") @@ -141,7 +156,7 @@ prepcompress() { # Queue up for compression. # ecompress{,dir} doesn't like to be called with empty argument lists. - [[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --queue "${incl_d[@]}" + [[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --limit ${PORTAGE_DOCOMPRESS_SIZE_LIMIT:-0} --queue "${incl_d[@]}" [[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${ED}}" [[ ${#exclude[@]} -gt 0 ]] && ecompressdir --ignore "${exclude[@]}" return 0 @@ -149,13 +164,12 @@ prepcompress() { install_qa_check() { local f i qa_var x - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local EPREFIX= ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local EPREFIX= ED=${D} + fi cd "${ED}" || die "cd failed" - # Merge QA_FLAGS_IGNORED and QA_DT_HASH into a single array, since - # QA_DT_HASH is deprecated. qa_var="QA_FLAGS_IGNORED_${ARCH/-/_}" eval "[[ -n \${!qa_var} ]] && QA_FLAGS_IGNORED=(\"\${${qa_var}[@]}\")" if [[ ${#QA_FLAGS_IGNORED[@]} -eq 1 ]] ; then @@ -166,29 +180,6 @@ install_qa_check() { set -${shopts} fi - qa_var="QA_DT_HASH_${ARCH/-/_}" - eval "[[ -n \${!qa_var} ]] && QA_DT_HASH=(\"\${${qa_var}[@]}\")" - if [[ ${#QA_DT_HASH[@]} -eq 1 ]] ; then - local shopts=$- - set -o noglob - QA_DT_HASH=(${QA_DT_HASH}) - set +o noglob - set -${shopts} - fi - - if [[ -n ${QA_DT_HASH} ]] ; then - QA_FLAGS_IGNORED=("${QA_FLAGS_IGNORED[@]}" "${QA_DT_HASH[@]}") - unset QA_DT_HASH - fi - - # Merge QA_STRICT_FLAGS_IGNORED and QA_STRICT_DT_HASH, since - # QA_STRICT_DT_HASH is deprecated - if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] && \ - [ "${QA_STRICT_DT_HASH-unset}" != unset ] ; then - QA_STRICT_FLAGS_IGNORED=1 - unset QA_STRICT_DT_HASH - fi - # Check for files built without respecting *FLAGS. Note that # -frecord-gcc-switches must be in all *FLAGS variables, in # order to avoid false positive results here. @@ -200,8 +191,7 @@ install_qa_check() { [[ "${FFLAGS}" == *-frecord-gcc-switches* ]] && \ [[ "${FCFLAGS}" == *-frecord-gcc-switches* ]] ; then rm -f "${T}"/scanelf-ignored-CFLAGS.log - for x in $(scanelf -qyRF '%k %p' -k \!.GCC.command.line "${ED}" | \ - sed -e "s:\!.GCC.command.line ::") ; do + for x in $(scanelf -qyRF '#k%p' -k '!.GCC.command.line' "${ED}") ; do # Separate out file types that are known to support # .GCC.command.line sections, using the `file` command # similar to how prepstrip uses it. @@ -226,11 +216,11 @@ install_qa_check() { -i "${T}"/scanelf-ignored-CFLAGS.log f=$(<"${T}"/scanelf-ignored-CFLAGS.log) if [[ -n ${f} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "${BAD}QA Notice: Files built without respecting CFLAGS have been detected${NORMAL}" eqawarn " Please include the following list of files in your report:" eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' sleep 1 else rm -f "${T}"/scanelf-ignored-CFLAGS.log @@ -240,7 +230,7 @@ install_qa_check() { export STRIP_MASK prepall - has "${EAPI}" 0 1 2 3 || prepcompress + ___eapi_has_docompress && prepcompress ecompressdir --dequeue ecompress --dequeue @@ -251,32 +241,50 @@ install_qa_check() { for x in etc/app-defaults usr/man usr/info usr/X11R6 usr/doc usr/locale ; do [[ -d ${ED}/$x ]] && f+=" $x\n" done - if [[ -n $f ]] ; then eqawarn "QA Notice: This ebuild installs into the following deprecated directories:" eqawarn eqawarn "$f" fi - if [[ -d ${ED}/etc/udev/rules.d ]] ; then - f= - for x in $(ls "${ED}/etc/udev/rules.d") ; do - f+=" etc/udev/rules.d/$x\n" - done - if [[ -n $f ]] ; then - eqawarn "QA Notice: udev rules should be installed in /lib/udev/rules.d:" - eqawarn - eqawarn "$f" + # It's ok create these directories, but not to install into them. #493154 + # TODO: We should add var/lib to this list. + f= + for x in var/cache var/lock var/run run ; do + if [[ ! -L ${ED}/${x} && -d ${ED}/${x} ]] ; then + if [[ -z $(find "${ED}/${x}" -prune -empty) ]] ; then + f+=$(cd "${ED}"; find "${x}" -printf ' %p\n') + fi fi + done + if [[ -n ${f} ]] ; then + eqawarn "QA Notice: This ebuild installs into paths that should be created at runtime." + eqawarn " To fix, simply do not install into these directories. Instead, your package" + eqawarn " should create dirs on the fly at runtime as needed via init scripts/etc..." + eqawarn + eqawarn "${f}" + fi + + set +f + f= + for x in "${ED}etc/udev/rules.d/"* "${ED}lib"*"/udev/rules.d/"* ; do + [[ -e ${x} ]] || continue + [[ ${x} == ${ED}lib/udev/rules.d/* ]] && continue + f+=" ${x#${ED}}\n" + done + if [[ -n $f ]] ; then + eqawarn "QA Notice: udev rules should be installed in /lib/udev/rules.d:" + eqawarn + eqawarn "$f" fi # Now we look for all world writable files. local unsafe_files=$(find "${ED}" -type f -perm -2 | sed -e "s:^${ED}:- :") if [[ -n ${unsafe_files} ]] ; then - vecho "QA Security Notice: world writable file(s):" - vecho "${unsafe_files}" - vecho "- This may or may not be a security problem, most of the time it is one." - vecho "- Please double check that $PF really needs a world writeable bit and file bugs accordingly." + __vecho "QA Security Notice: world writable file(s):" + __vecho "${unsafe_files}" + __vecho "- This may or may not be a security problem, most of the time it is one." + __vecho "- Please double check that $PF really needs a world writeable bit and file bugs accordingly." sleep 1 fi @@ -307,7 +315,7 @@ install_qa_check() { for l in $(echo "${rpath_files}" | grep -E ":${dir}|::|: "); do f+=" ${l%%:*}\n" if ! has stricter ${FEATURES}; then - vecho "Auto fixing rpaths for ${l%%:*}" + __vecho "Auto fixing rpaths for ${l%%:*}" TMPDIR="${dir}" scanelf -BXr "${l%%:*}" -o /dev/null fi done @@ -321,12 +329,12 @@ install_qa_check() { # Print QA notice. if [[ -n ${f}${x} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: The following files contain insecure RUNPATHs" eqawarn " Please file a bug about this at http://bugs.gentoo.org/" eqawarn " with the maintaining herd of the package." eqawarn "${f}${f:+${x:+\n}}${x}" - vecho -ne '\n' + __vecho -ne '\n' if [[ -n ${x} ]] || has stricter ${FEATURES} ; then insecure_rpath=1 fi @@ -344,7 +352,7 @@ install_qa_check() { f=$(scanelf -qyRF '%t %p' "${ED}" | grep -v 'usr/lib/debug/') if [[ -n ${f} ]] ; then scanelf -qyRAF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: The following files contain runtime text relocations" eqawarn " Text relocations force the dynamic linker to perform extra" eqawarn " work at startup, waste system resources, and may pose a security" @@ -353,7 +361,7 @@ install_qa_check() { eqawarn " For more information, see http://hardened.gentoo.org/pic-fix-guide.xml" eqawarn " Please include the following list of files in your report:" eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' die_msg="${die_msg} textrels," sleep 1 fi @@ -364,7 +372,7 @@ install_qa_check() { *-linux-gnu*) # Check for files with executable stacks, but only on arches which # are supported at the moment. Keep this list in sync with - # http://hardened.gentoo.org/gnu-stack.xml (Arch Status) + # http://www.gentoo.org/proj/en/hardened/gnu-stack.xml (Arch Status) case ${CTARGET:-${CHOST}} in arm*|i?86*|ia64*|m68k*|s390*|sh*|x86_64*) # Allow devs to mark things as ignorable ... e.g. things @@ -389,7 +397,7 @@ install_qa_check() { if [[ -n ${f} ]] ; then # One more pass to help devs track down the source scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: The following files contain writable and executable sections" eqawarn " Files with such sections will not work properly (or at all!) on some" eqawarn " architectures/operating systems. A bug should be filed at" @@ -399,15 +407,15 @@ install_qa_check() { eqawarn " Note: Bugs should be filed for the respective maintainers" eqawarn " of the package in question and not hardened@g.o." eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' die_msg="${die_msg} execstacks" sleep 1 fi # Check for files built without respecting LDFLAGS if [[ "${LDFLAGS}" == *,--hash-style=gnu* ]] && \ - ! has binchecks ${RESTRICT} ; then - f=$(scanelf -qyRF '%k %p' -k .hash "${ED}" | sed -e "s:\.hash ::") + ! has binchecks ${RESTRICT} ; then + f=$(scanelf -qyRF '#k%p' -k .hash "${ED}") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then @@ -421,11 +429,11 @@ install_qa_check() { -i "${T}"/scanelf-ignored-LDFLAGS.log f=$(<"${T}"/scanelf-ignored-LDFLAGS.log) if [[ -n ${f} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "${BAD}QA Notice: Files built without respecting LDFLAGS have been detected${NORMAL}" eqawarn " Please include the following list of files in your report:" eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' sleep 1 else rm -f "${T}"/scanelf-ignored-LDFLAGS.log @@ -442,7 +450,7 @@ install_qa_check() { # Check for shared libraries lacking SONAMEs qa_var="QA_SONAME_${ARCH/-/_}" eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")" - f=$(scanelf -ByF '%S %p' "${ED}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") + f=$(scanelf -ByF '%S %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-missing-SONAME.log if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then @@ -463,10 +471,10 @@ install_qa_check() { sed -e "/^\$/d" -i "${T}"/scanelf-missing-SONAME.log f=$(<"${T}"/scanelf-missing-SONAME.log) if [[ -n ${f} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: The following shared libraries lack a SONAME" eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' sleep 1 else rm -f "${T}"/scanelf-missing-SONAME.log @@ -476,7 +484,7 @@ install_qa_check() { # Check for shared libraries lacking NEEDED entries qa_var="QA_DT_NEEDED_${ARCH/-/_}" eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")" - f=$(scanelf -ByF '%n %p' "${ED}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") + f=$(scanelf -ByF '%n %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-missing-NEEDED.log if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then @@ -497,10 +505,10 @@ install_qa_check() { sed -e "/^\$/d" -i "${T}"/scanelf-missing-NEEDED.log f=$(<"${T}"/scanelf-missing-NEEDED.log) if [[ -n ${f} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: The following shared libraries lack NEEDED entries" eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' sleep 1 else rm -f "${T}"/scanelf-missing-NEEDED.log @@ -545,14 +553,13 @@ install_qa_check() { die "Unsafe files found in \${D}. Portage will not install them." fi - if [[ -d ${D}/${D} ]] ; then - declare -i INSTALLTOD=0 - for i in $(find "${D}/${D}/"); do - eqawarn "QA Notice: /${i##${D}/${D}} installed in \${D}/\${D}" + if [[ -d ${D%/}${D} ]] ; then + local -i INSTALLTOD=0 + while read -r -d $'\0' i ; do + eqawarn "QA Notice: /${i##${D%/}${D}} installed in \${D}/\${D}" ((INSTALLTOD++)) - done - die "Aborting due to QA concerns: ${INSTALLTOD} files installed in ${D}/${D}" - unset INSTALLTOD + done < <(find "${D%/}${D}" -print0) + die "Aborting due to QA concerns: ${INSTALLTOD} files installed in ${D%/}${D}" fi # Sanity check syntax errors in init.d scripts @@ -563,10 +570,31 @@ install_qa_check() { [[ -L ${i} ]] && continue # if empty conf.d/init.d dir exists (baselayout), then i will be "/etc/conf.d/*" and not exist [[ ! -e ${i} ]] && continue + if [[ ${d} == /etc/init.d && ${i} != *.sh ]] ; then + # skip non-shell-script for bug #451386 + [[ $(head -n1 "${i}") =~ ^#!.*[[:space:]/](runscript|sh)$ ]] || continue + fi bash -n "${i}" || die "The init.d file has syntax errors: ${i}" done done + local checkbashisms=$(type -P checkbashisms) + if [[ -n ${checkbashisms} ]] ; then + for d in /etc/init.d ; do + [[ -d ${ED}${d} ]] || continue + for i in "${ED}${d}"/* ; do + [[ -e ${i} ]] || continue + [[ -L ${i} ]] && continue + f=$("${checkbashisms}" -f "${i}" 2>&1) + [[ $? != 0 && -n ${f} ]] || continue + eqawarn "QA Notice: shell script appears to use non-POSIX feature(s):" + while read -r ; + do eqawarn " ${REPLY}" + done <<< "${f//${ED}}" + done + done + fi + # Look for leaking LDFLAGS into pkg-config files f=$(egrep -sH '^Libs.*-Wl,(-O[012]|--hash-style)' "${ED}"/usr/*/pkgconfig/*.pc) if [[ -n ${f} ]] ; then @@ -577,17 +605,16 @@ install_qa_check() { # this should help to ensure that all (most?) shared libraries are executable # and that all libtool scripts / static libraries are not executable local j - for i in "${ED}"opt/*/lib{,32,64} \ - "${ED}"lib{,32,64} \ - "${ED}"usr/lib{,32,64} \ - "${ED}"usr/X11R6/lib{,32,64} ; do + for i in "${ED}"opt/*/lib* \ + "${ED}"lib* \ + "${ED}"usr/lib* ; do [[ ! -d ${i} ]] && continue for j in "${i}"/*.so.* "${i}"/*.so ; do [[ ! -e ${j} ]] && continue [[ -L ${j} ]] && continue [[ -x ${j} ]] && continue - vecho "making executable: ${j#${ED}}" + __vecho "making executable: ${j#${ED}}" chmod +x "${j}" done @@ -595,7 +622,7 @@ install_qa_check() { [[ ! -e ${j} ]] && continue [[ -L ${j} ]] && continue [[ ! -x ${j} ]] && continue - vecho "removing executable bit: ${j#${ED}}" + __vecho "removing executable bit: ${j#${ED}}" chmod -x "${j}" done @@ -604,7 +631,7 @@ install_qa_check() { [[ ! -L ${j} ]] && continue linkdest=$(readlink "${j}") if [[ ${linkdest} == /* ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: Found an absolute symlink in a library directory:" eqawarn " ${j#${D}} -> ${linkdest}" eqawarn " It should be a relative symlink if in the same directory" @@ -613,8 +640,8 @@ install_qa_check() { done done - # When installing static libraries into /usr/lib and shared libraries into - # /lib, we have to make sure we have a linker script in /usr/lib along side + # When installing static libraries into /usr/lib and shared libraries into + # /lib, we have to make sure we have a linker script in /usr/lib along side # the static library, or gcc will utilize the static lib when linking :(. # http://bugs.gentoo.org/4411 abort="no" @@ -624,7 +651,7 @@ install_qa_check() { if [[ ! -e ${s} ]] ; then s=${s%usr/*}${s##*/usr/} if [[ -e ${s} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: Missing gen_usr_ldscript for ${s##*/}" abort="yes" fi @@ -635,11 +662,11 @@ install_qa_check() { # Make sure people don't store libtool files or static libs in /lib f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) if [[ -n ${f} ]] ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: Excessive files found in the / partition" eqawarn "${f}" - vecho -ne '\n' - die "static archives (*.a) and libtool library files (*.la) do not belong in /" + __vecho -ne '\n' + die "static archives (*.a) and libtool library files (*.la) belong in /usr/lib*, not /lib*" fi # Verify that the libtool files don't contain bogus $D entries. @@ -647,7 +674,7 @@ install_qa_check() { for a in "${ED}"usr/lib*/*.la ; do s=${a##*/} if grep -qs "${ED}" "${a}" ; then - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" abort="yes" fi @@ -688,6 +715,8 @@ install_qa_check() { ": warning: reference to local variable .* returned" ": warning: returning reference to temporary" ": warning: function returns address of local variable" + ": warning: .*\\[-Wsizeof-pointer-memaccess\\]" + ": warning: .*\\[-Waggressive-loop-optimizations\\]" # this may be valid code :/ #": warning: multi-character character constant" # need to check these two ... @@ -726,18 +755,19 @@ install_qa_check() { eerror " with the maintaining herd of the package." eerror else - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: Package triggers severe warnings which indicate that it" eqawarn " may exhibit random runtime failures." eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' fi fi done local cat_cmd=cat [[ $PORTAGE_LOG_FILE = *.gz ]] && cat_cmd=zcat [[ $reset_debug = 1 ]] && set -x - f=$($cat_cmd "${PORTAGE_LOG_FILE}" | \ + # Use safe cwd, avoiding unsafe import for bug #469338. + f=$(cd "${PORTAGE_PYM_PATH}" ; $cat_cmd "${PORTAGE_LOG_FILE}" | \ "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/check-implicit-pointer-usage.py || die "check-implicit-pointer-usage.py failed") if [[ -n ${f} ]] ; then @@ -763,26 +793,25 @@ install_qa_check() { eerror " with the maintaining herd of the package." eerror else - vecho -ne '\n' + __vecho -ne '\n' eqawarn "QA Notice: Package triggers severe warnings which indicate that it" eqawarn " will almost certainly crash on 64bit architectures." eqawarn "${f}" - vecho -ne '\n' + __vecho -ne '\n' fi fi if [[ ${abort} == "yes" ]] ; then if [[ $gentoo_bug = yes || $always_overflow = yes ]] ; then - die "install aborted due to" \ - "severe warnings shown above" + die "install aborted due to severe warnings shown above" else echo "Please do not file a Gentoo bug and instead" \ "report the above QA issues directly to the upstream" \ "developers of this software." | fmt -w 70 | \ while read -r line ; do eqawarn "${line}" ; done eqawarn "Homepage: ${HOMEPAGE}" - has stricter ${FEATURES} && die "install aborted due to" \ - "severe warnings shown above" + has stricter ${FEATURES} && \ + die "install aborted due to severe warnings shown above" fi fi fi @@ -794,32 +823,42 @@ install_qa_check() { [[ -x /usr/bin/file && -x /usr/bin/find ]] && \ [[ -n ${MULTILIB_STRICT_DIRS} && -n ${MULTILIB_STRICT_DENY} ]] then - local abort=no dir file firstrun=yes + rm -f "${T}/multilib-strict.log" + local abort=no dir file MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT} | sed -e 's:\([(|)]\):\\\1:g') for dir in ${MULTILIB_STRICT_DIRS} ; do [[ -d ${ED}/${dir} ]] || continue for file in $(find ${ED}/${dir} -type f | grep -v "^${ED}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do if file ${file} | egrep -q "${MULTILIB_STRICT_DENY}" ; then - if [[ ${firstrun} == yes ]] ; then - echo "Files matching a file type that is not allowed:" - firstrun=no - fi - abort=yes - echo " ${file#${ED}//}" + echo "${file#${ED}//}" >> "${T}/multilib-strict.log" fi done done - [[ ${abort} == yes ]] && die "multilib-strict check failed!" - fi - # ensure packages don't install systemd units automagically - if ! has systemd ${INHERITED} && \ - [[ -d "${ED}"/lib/systemd/system ]] - then - eqawarn "QA Notice: package installs systemd unit files (/lib/systemd/system)" - eqawarn " but does not inherit systemd.eclass." - has stricter ${FEATURES} \ - && die "install aborted due to missing inherit of systemd.eclass" + if [[ -s ${T}/multilib-strict.log ]] ; then + if [[ ${#QA_MULTILIB_PATHS[@]} -eq 1 ]] ; then + local shopts=$- + set -o noglob + QA_MULTILIB_PATHS=(${QA_MULTILIB_PATHS}) + set +o noglob + set -${shopts} + fi + if [ "${QA_STRICT_MULTILIB_PATHS-unset}" = unset ] ; then + for x in "${QA_MULTILIB_PATHS[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}/multilib-strict.log" + done + sed -e "/^\$/d" -i "${T}/multilib-strict.log" + fi + if [[ -s ${T}/multilib-strict.log ]] ; then + abort=yes + echo "Files matching a file type that is not allowed:" + while read -r ; do + echo " ${REPLY}" + done < "${T}/multilib-strict.log" + fi + fi + + [[ ${abort} == yes ]] && die "multilib-strict check failed!" fi } @@ -852,16 +891,6 @@ install_qa_check_prefix() { # all further checks rely on ${ED} existing [[ -d ${ED} ]] || return - # this does not really belong here, but it's closely tied to - # the code below; many runscripts generate positives here, and we - # know they don't work (bug #196294) so as long as that one - # remains an issue, simply remove them as they won't work - # anyway, avoid etc/init.d/functions.sh from being thrown away - if [[ ( -d "${ED}"/etc/conf.d || -d "${ED}"/etc/init.d ) && ! -f "${ED}"/etc/init.d/functions.sh ]] ; then - ewarn "removed /etc/init.d and /etc/conf.d directories until bug #196294 has been resolved" - rm -Rf "${ED}"/etc/{conf,init}.d - fi - # check shebangs, bug #282539 rm -f "${T}"/non-prefix-shebangs-errs local WHITELIST=" /usr/bin/env " @@ -953,7 +982,7 @@ install_mask() { local no_inst for no_inst in ${install_mask}; do set +o noglob - quiet_mode || einfo "Removing ${no_inst}" + __quiet_mode || einfo "Removing ${no_inst}" # normal stuff rm -Rf "${root}"/${no_inst} >&/dev/null @@ -972,8 +1001,9 @@ preinst_mask() { return 1 fi - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi # Make sure $PWD is not ${D} so that we don't leave gmon.out files # in there in case any tools were built with -pg in CFLAGS. @@ -1001,8 +1031,9 @@ preinst_sfperms() { return 1 fi - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi # Smart FileSystem Permissions if has sfperms $FEATURES; then @@ -1040,8 +1071,9 @@ preinst_suid_scan() { return 1 fi - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi # total suid control. if has suidctl $FEATURES; then @@ -1051,19 +1083,19 @@ preinst_suid_scan() { # to files outside of the sandbox, but this # can easly be bypassed using the addwrite() function addwrite "${sfconf}" - vecho ">>> Performing suid scan in ${ED}" + __vecho ">>> Performing suid scan in ${ED}" for i in $(find "${ED}" -type f \( -perm -4000 -o -perm -2000 \) ); do if [ -s "${sfconf}" ]; then install_path=/${i#${ED}} if grep -q "^${install_path}\$" "${sfconf}" ; then - vecho "- ${install_path} is an approved suid file" + __vecho "- ${install_path} is an approved suid file" else - vecho ">>> Removing sbit on non registered ${install_path}" + __vecho ">>> Removing sbit on non registered ${install_path}" for x in 5 4 3 2 1 0; do sleep 0.25 ; done ls_ret=$(ls -ldh "${i}") chmod ugo-s "${i}" grep "^#${install_path}$" "${sfconf}" > /dev/null || { - vecho ">>> Appending commented out entry to ${sfconf} for ${PF}" + __vecho ">>> Appending commented out entry to ${sfconf} for ${PF}" echo "## ${ls_ret%${ED}*}${install_path}" >> "${sfconf}" echo "#${install_path}" >> "${sfconf}" # no delwrite() eh? @@ -1071,7 +1103,7 @@ preinst_suid_scan() { } fi else - vecho "suidctl feature set but you are lacking a ${sfconf}" + __vecho "suidctl feature set but you are lacking a ${sfconf}" fi done fi @@ -1083,34 +1115,35 @@ preinst_selinux_labels() { return 1 fi if has selinux ${FEATURES}; then - # SELinux file labeling (needs to always be last in dyn_preinst) + # SELinux file labeling (needs to execute after preinst) # only attempt to label if setfiles is executable # and 'context' is available on selinuxfs. if [ -f /selinux/context -o -f /sys/fs/selinux/context ] && \ [ -x /usr/sbin/setfiles -a -x /usr/sbin/selinuxconfig ]; then - vecho ">>> Setting SELinux security labels" + __vecho ">>> Setting SELinux security labels" ( eval "$(/usr/sbin/selinuxconfig)" || \ die "Failed to determine SELinux policy paths."; - + addwrite /selinux/context addwrite /sys/fs/selinux/context - + /usr/sbin/setfiles "${file_contexts_path}" -r "${D}" "${D}" ) || die "Failed to set SELinux security labels." else # nonfatal, since merging can happen outside a SE kernel # like during a recovery situation - vecho "!!! Unable to set SELinux security labels" + __vecho "!!! Unable to set SELinux security labels" fi fi } -dyn_package() { +__dyn_package() { local PROOT - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local EPREFIX= ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local EPREFIX= ED=${D} + fi # Make sure $PWD is not ${D} so that we don't leave gmon.out files # in there in case any tools were built with -pg in CFLAGS. @@ -1133,6 +1166,7 @@ dyn_package() { local tar_options="" [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v" + has xattr ${FEATURES} && [[ $(tar --help 2> /dev/null) == *--xattrs* ]] && tar_options+=" --xattrs" # Sandbox is disabled in case the user wants to use a symlink # for $PKGDIR and/or $PKGDIR/All. export SANDBOX_ON="0" @@ -1142,7 +1176,7 @@ dyn_package() { tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${PROOT}" . | \ $PORTAGE_BZIP2_COMMAND -c > "$PORTAGE_BINPKG_TMPFILE" assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'" - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \ "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info" if [ $? -ne 0 ]; then @@ -1159,7 +1193,7 @@ dyn_package() { fi [ -n "${md5_hash}" ] && \ echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5 - vecho ">>> Done." + __vecho ">>> Done." # cleanup our temp tree [[ -n ${PKG_INSTALL_MASK} ]] && rm -rf "${PROOT}" @@ -1168,8 +1202,8 @@ dyn_package() { die "Failed to create $PORTAGE_BUILDDIR/.packaged" } -dyn_spec() { - local sources_dir=/usr/src/rpm/SOURCES +__dyn_spec() { + local sources_dir=${T}/rpmbuild/SOURCES mkdir -p "${sources_dir}" declare -a tar_args=("${EBUILD}") [[ -d ${FILESDIR} ]] && tar_args=("${EBUILD}" "${FILESDIR}") @@ -1182,10 +1216,9 @@ Summary: ${DESCRIPTION} Name: ${PN} Version: ${PV} Release: ${PR} -Copyright: GPL +License: GPL Group: portage/${CATEGORY} Source: ${PF}.tar.gz -Buildroot: ${D} %description ${DESCRIPTION} @@ -1206,18 +1239,18 @@ __END1__ } -dyn_rpm() { - - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local EPREFIX= ;; esac +__dyn_rpm() { + if ! ___eapi_has_prefix_variables; then + local EPREFIX= + fi cd "${T}" || die "cd failed" - local machine_name=$(uname -m) - local dest_dir=${EPREFIX}/usr/src/rpm/RPMS/${machine_name} - addwrite ${EPREFIX}/usr/src/rpm + local machine_name=${CHOST%%-*} + local dest_dir=${T}/rpmbuild/RPMS/${machine_name} addwrite "${RPMDIR}" - dyn_spec - rpmbuild -bb --clean --rmsource "${PF}.spec" || die "Failed to integrate rpm spec file" + __dyn_spec + HOME=${T} \ + rpmbuild -bb --clean --nodeps --rmsource "${PF}.spec" --buildroot "${D}" --target "${CHOST}" || die "Failed to integrate rpm spec file" install -D "${dest_dir}/${PN}-${PV}-${PR}.${machine_name}.rpm" \ "${RPMDIR}/${CATEGORY}/${PN}-${PV}-${PR}.rpm" || \ die "Failed to move rpm" @@ -1255,7 +1288,7 @@ install_hooks() { } if [ -n "${MISC_FUNCTIONS_ARGS}" ]; then - source_all_bashrcs + __source_all_bashrcs [ "$PORTAGE_DEBUG" == "1" ] && set -x for x in ${MISC_FUNCTIONS_ARGS}; do ${x} diff --git a/portage_with_autodep/bin/phase-functions.sh b/portage_with_autodep/bin/phase-functions.sh index ce251ce..f39a024 100755 --- a/portage_with_autodep/bin/phase-functions.sh +++ b/portage_with_autodep/bin/phase-functions.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Hardcoded bash lists are needed for backward compatibility with @@ -8,28 +8,31 @@ # when portage is upgrading itself. PORTAGE_READONLY_METADATA="DEFINED_PHASES DEPEND DESCRIPTION - EAPI HOMEPAGE INHERITED IUSE REQUIRED_USE KEYWORDS LICENSE + EAPI HDEPEND HOMEPAGE INHERITED IUSE REQUIRED_USE KEYWORDS LICENSE PDEPEND PROVIDE RDEPEND REPOSITORY RESTRICT SLOT SRC_URI" -PORTAGE_READONLY_VARS="D EBUILD EBUILD_PHASE \ +PORTAGE_READONLY_VARS="D EBUILD EBUILD_PHASE EBUILD_PHASE_FUNC \ EBUILD_SH_ARGS ECLASSDIR EMERGE_FROM FILESDIR MERGE_TYPE \ PM_EBUILD_HOOK_DIR \ PORTAGE_ACTUAL_DISTDIR PORTAGE_ARCHLIST PORTAGE_BASHRC \ PORTAGE_BINPKG_FILE PORTAGE_BINPKG_TAR_OPTS PORTAGE_BINPKG_TMPFILE \ - PORTAGE_BIN_PATH PORTAGE_BUILDDIR PORTAGE_BUNZIP2_COMMAND \ + PORTAGE_BIN_PATH PORTAGE_BUILDDIR PORTAGE_BUILD_GROUP \ + PORTAGE_BUILD_USER PORTAGE_BUNZIP2_COMMAND \ PORTAGE_BZIP2_COMMAND PORTAGE_COLORMAP PORTAGE_CONFIGROOT \ PORTAGE_DEBUG PORTAGE_DEPCACHEDIR PORTAGE_EBUILD_EXIT_FILE \ + PORTAGE_ECLASS_LOCATIONS \ PORTAGE_GID PORTAGE_GRPNAME PORTAGE_INST_GID PORTAGE_INST_UID \ - PORTAGE_IPC_DAEMON PORTAGE_IUSE PORTAGE_LOG_FILE \ + PORTAGE_INTERNAL_CALLER PORTAGE_IPC_DAEMON PORTAGE_IUSE PORTAGE_LOG_FILE \ PORTAGE_MUTABLE_FILTERED_VARS PORTAGE_OVERRIDE_EPREFIX \ - PORTAGE_PYM_PATH PORTAGE_PYTHON \ + PORTAGE_PYM_PATH PORTAGE_PYTHON PORTAGE_PYTHONPATH \ PORTAGE_READONLY_METADATA PORTAGE_READONLY_VARS \ - PORTAGE_REPO_NAME PORTAGE_RESTRICT \ + PORTAGE_REPO_NAME PORTAGE_REPOSITORIES PORTAGE_RESTRICT \ PORTAGE_SAVED_READONLY_VARS PORTAGE_SIGPIPE_STATUS \ PORTAGE_TMPDIR PORTAGE_UPDATE_ENV PORTAGE_USERNAME \ - PORTAGE_VERBOSE PORTAGE_WORKDIR_MODE PORTDIR PORTDIR_OVERLAY \ + PORTAGE_VERBOSE PORTAGE_WORKDIR_MODE PORTAGE_XATTR_EXCLUDE \ + PORTDIR \ PROFILE_PATHS REPLACING_VERSIONS REPLACED_BY_VERSION T WORKDIR \ - __PORTAGE_TEST_HARDLINK_LOCKS" + __PORTAGE_HELPER __PORTAGE_TEST_HARDLINK_LOCKS" PORTAGE_SAVED_READONLY_VARS="A CATEGORY P PF PN PR PV PVR" @@ -39,7 +42,7 @@ PORTAGE_SAVED_READONLY_VARS="A CATEGORY P PF PN PR PV PVR" # it is saved or loaded (any mutations do not persist). PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" -# @FUNCTION: filter_readonly_variables +# @FUNCTION: __filter_readonly_variables # @DESCRIPTION: [--filter-sandbox] [--allow-extra-vars] # Read an environment from stdin and echo to stdout while filtering variables # with names that are known to cause interference: @@ -81,14 +84,14 @@ PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" # readonly variable cause the shell to exit while executing the "source" # builtin command. To avoid this problem, this function filters those # variables out and discards them. See bug #190128. -filter_readonly_variables() { +__filter_readonly_variables() { local x filtered_vars local readonly_bash_vars="BASHOPTS BASHPID DIRSTACK EUID FUNCNAME GROUPS PIPESTATUS PPID SHELLOPTS UID" local bash_misc_vars="BASH BASH_.* COLUMNS COMP_WORDBREAKS HISTCMD HISTFILE HOSTNAME HOSTTYPE IFS LINENO MACHTYPE OLDPWD OPTERR OPTIND OSTYPE POSIXLY_CORRECT PS4 PWD RANDOM - SECONDS SHELL SHLVL _" + SECONDS SHLVL _" local filtered_sandbox_vars="SANDBOX_ACTIVE SANDBOX_BASHRC SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB SANDBOX_LOG SANDBOX_ON" @@ -100,15 +103,9 @@ filter_readonly_variables() { # Don't filter/interfere with prefix variables unless they are # supported by the current EAPI. - case "${EAPI:-0}" in - 0|1|2) - [[ " ${FEATURES} " == *" force-prefix "* ]] && \ - filtered_vars+=" ED EPREFIX EROOT" - ;; - *) - filtered_vars+=" ED EPREFIX EROOT" - ;; - esac + if ___eapi_has_prefix_variables; then + filtered_vars+=" ED EPREFIX EROOT" + fi if has --filter-sandbox $* ; then filtered_vars="${filtered_vars} SANDBOX_.*" @@ -140,14 +137,14 @@ filter_readonly_variables() { "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}" || die "filter-bash-environment.py failed" } -# @FUNCTION: preprocess_ebuild_env +# @FUNCTION: __preprocess_ebuild_env # @DESCRIPTION: # Filter any readonly variables from ${T}/environment, source it, and then -# save it via save_ebuild_env(). This process should be sufficient to prevent +# save it via __save_ebuild_env(). This process should be sufficient to prevent # any stale variables or functions from an arbitrary environment from # interfering with the current environment. This is useful when an existing # environment needs to be loaded from a binary or installed package. -preprocess_ebuild_env() { +__preprocess_ebuild_env() { local _portage_filter_opts="--filter-features --filter-locale --filter-path --filter-sandbox" # If environment.raw is present, this is a signal from the python side, @@ -156,7 +153,7 @@ preprocess_ebuild_env() { # Otherwise, we don't need to filter the environment. [ -f "${T}/environment.raw" ] || return 0 - filter_readonly_variables $_portage_filter_opts < "${T}"/environment \ + __filter_readonly_variables $_portage_filter_opts < "${T}"/environment \ >> "$T/environment.filtered" || return $? unset _portage_filter_opts mv "${T}"/environment.filtered "${T}"/environment || return $? @@ -174,20 +171,20 @@ preprocess_ebuild_env() { # until we've merged them with our current values. export SANDBOX_ON=0 - # It's remotely possible that save_ebuild_env() has been overridden + # It's remotely possible that __save_ebuild_env() has been overridden # by the above source command. To protect ourselves, we override it # here with our own version. ${PORTAGE_BIN_PATH} is safe to use here # because it's already filtered above. source "${PORTAGE_BIN_PATH}/save-ebuild-env.sh" || exit $? - # Rely on save_ebuild_env() to filter out any remaining variables + # Rely on __save_ebuild_env() to filter out any remaining variables # and functions that could interfere with the current environment. - save_ebuild_env || exit $? + __save_ebuild_env || exit $? >> "$T/environment.success" || exit $? ) > "${T}/environment.filtered" local retval if [ -e "${T}/environment.success" ] ; then - filter_readonly_variables --filter-features < \ + __filter_readonly_variables --filter-features < \ "${T}/environment.filtered" > "${T}/environment" retval=$? else @@ -197,62 +194,62 @@ preprocess_ebuild_env() { return ${retval} } -ebuild_phase() { - declare -F "$1" >/dev/null && qa_call $1 +__ebuild_phase() { + declare -F "$1" >/dev/null && __qa_call $1 } -ebuild_phase_with_hooks() { +__ebuild_phase_with_hooks() { local x phase_name=${1} for x in {pre_,,post_}${phase_name} ; do - ebuild_phase ${x} + __ebuild_phase ${x} done } -dyn_pretend() { +__dyn_pretend() { if [[ -e $PORTAGE_BUILDDIR/.pretended ]] ; then - vecho ">>> It appears that '$PF' is already pretended; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.pretended' to force pretend." + __vecho ">>> It appears that '$PF' is already pretended; skipping." + __vecho ">>> Remove '$PORTAGE_BUILDDIR/.pretended' to force pretend." return 0 fi - ebuild_phase pre_pkg_pretend - ebuild_phase pkg_pretend + __ebuild_phase pre_pkg_pretend + __ebuild_phase pkg_pretend >> "$PORTAGE_BUILDDIR/.pretended" || \ die "Failed to create $PORTAGE_BUILDDIR/.pretended" - ebuild_phase post_pkg_pretend + __ebuild_phase post_pkg_pretend } -dyn_setup() { +__dyn_setup() { if [[ -e $PORTAGE_BUILDDIR/.setuped ]] ; then - vecho ">>> It appears that '$PF' is already setup; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.setuped' to force setup." + __vecho ">>> It appears that '$PF' is already setup; skipping." + __vecho ">>> Remove '$PORTAGE_BUILDDIR/.setuped' to force setup." return 0 fi - ebuild_phase pre_pkg_setup - ebuild_phase pkg_setup + __ebuild_phase pre_pkg_setup + __ebuild_phase pkg_setup >> "$PORTAGE_BUILDDIR/.setuped" || \ die "Failed to create $PORTAGE_BUILDDIR/.setuped" - ebuild_phase post_pkg_setup + __ebuild_phase post_pkg_setup } -dyn_unpack() { +__dyn_unpack() { if [[ -f ${PORTAGE_BUILDDIR}/.unpacked ]] ; then - vecho ">>> WORKDIR is up-to-date, keeping..." + __vecho ">>> WORKDIR is up-to-date, keeping..." return 0 fi if [ ! -d "${WORKDIR}" ]; then install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'" fi cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`" - ebuild_phase pre_src_unpack - vecho ">>> Unpacking source..." - ebuild_phase src_unpack + __ebuild_phase pre_src_unpack + __vecho ">>> Unpacking source..." + __ebuild_phase src_unpack >> "$PORTAGE_BUILDDIR/.unpacked" || \ die "Failed to create $PORTAGE_BUILDDIR/.unpacked" - vecho ">>> Source unpacked in ${WORKDIR}" - ebuild_phase post_src_unpack + __vecho ">>> Source unpacked in ${WORKDIR}" + __ebuild_phase post_src_unpack } -dyn_clean() { +__dyn_clean() { if [ -z "${PORTAGE_BUILDDIR}" ]; then echo "Aborting clean phase because PORTAGE_BUILDDIR is unset!" return 1 @@ -299,7 +296,7 @@ dyn_clean() { true } -abort_handler() { +__abort_handler() { local msg if [ "$2" != "fail" ]; then msg="${EBUILD}: ${1} aborted; exiting." @@ -314,37 +311,37 @@ abort_handler() { trap - SIGINT SIGQUIT } -abort_prepare() { - abort_handler src_prepare $1 +__abort_prepare() { + __abort_handler src_prepare $1 rm -f "$PORTAGE_BUILDDIR/.prepared" exit 1 } -abort_configure() { - abort_handler src_configure $1 +__abort_configure() { + __abort_handler src_configure $1 rm -f "$PORTAGE_BUILDDIR/.configured" exit 1 } -abort_compile() { - abort_handler "src_compile" $1 +__abort_compile() { + __abort_handler "src_compile" $1 rm -f "${PORTAGE_BUILDDIR}/.compiled" exit 1 } -abort_test() { - abort_handler "dyn_test" $1 +__abort_test() { + __abort_handler "__dyn_test" $1 rm -f "${PORTAGE_BUILDDIR}/.tested" exit 1 } -abort_install() { - abort_handler "src_install" $1 +__abort_install() { + __abort_handler "src_install" $1 rm -rf "${PORTAGE_BUILDDIR}/image" exit 1 } -has_phase_defined_up_to() { +__has_phase_defined_up_to() { local phase for phase in unpack prepare configure compile install; do has ${phase} ${DEFINED_PHASES} && return 0 @@ -354,89 +351,89 @@ has_phase_defined_up_to() { return 1 } -dyn_prepare() { +__dyn_prepare() { if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then - vecho ">>> It appears that '$PF' is already prepared; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare." + __vecho ">>> It appears that '$PF' is already prepared; skipping." + __vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare." return 0 fi if [[ -d $S ]] ; then cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then + elif ___eapi_has_S_WORKDIR_fallback; then cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to prepare; then + elif [[ -z ${A} ]] && ! __has_phase_defined_up_to prepare; then cd "${WORKDIR}" else die "The source directory '${S}' doesn't exist" fi - trap abort_prepare SIGINT SIGQUIT + trap __abort_prepare SIGINT SIGQUIT - ebuild_phase pre_src_prepare - vecho ">>> Preparing source in $PWD ..." - ebuild_phase src_prepare + __ebuild_phase pre_src_prepare + __vecho ">>> Preparing source in $PWD ..." + __ebuild_phase src_prepare >> "$PORTAGE_BUILDDIR/.prepared" || \ die "Failed to create $PORTAGE_BUILDDIR/.prepared" - vecho ">>> Source prepared." - ebuild_phase post_src_prepare + __vecho ">>> Source prepared." + __ebuild_phase post_src_prepare trap - SIGINT SIGQUIT } -dyn_configure() { +__dyn_configure() { if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then - vecho ">>> It appears that '$PF' is already configured; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration." + __vecho ">>> It appears that '$PF' is already configured; skipping." + __vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration." return 0 fi if [[ -d $S ]] ; then cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then + elif ___eapi_has_S_WORKDIR_fallback; then cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to configure; then + elif [[ -z ${A} ]] && ! __has_phase_defined_up_to configure; then cd "${WORKDIR}" else die "The source directory '${S}' doesn't exist" fi - trap abort_configure SIGINT SIGQUIT + trap __abort_configure SIGINT SIGQUIT - ebuild_phase pre_src_configure + __ebuild_phase pre_src_configure - vecho ">>> Configuring source in $PWD ..." - ebuild_phase src_configure + __vecho ">>> Configuring source in $PWD ..." + __ebuild_phase src_configure >> "$PORTAGE_BUILDDIR/.configured" || \ die "Failed to create $PORTAGE_BUILDDIR/.configured" - vecho ">>> Source configured." + __vecho ">>> Source configured." - ebuild_phase post_src_configure + __ebuild_phase post_src_configure trap - SIGINT SIGQUIT } -dyn_compile() { +__dyn_compile() { if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then - vecho ">>> It appears that '${PF}' is already compiled; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation." + __vecho ">>> It appears that '${PF}' is already compiled; skipping." + __vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation." return 0 fi if [[ -d $S ]] ; then cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then + elif ___eapi_has_S_WORKDIR_fallback; then cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to compile; then + elif [[ -z ${A} ]] && ! __has_phase_defined_up_to compile; then cd "${WORKDIR}" else die "The source directory '${S}' doesn't exist" fi - trap abort_compile SIGINT SIGQUIT + trap __abort_compile SIGINT SIGQUIT if has distcc $FEATURES && has distcc-pump $FEATURES ; then if [[ -z $INCLUDE_SERVER_PORT ]] || [[ ! -w $INCLUDE_SERVER_PORT ]] ; then @@ -445,90 +442,96 @@ dyn_compile() { fi fi - ebuild_phase pre_src_compile + __ebuild_phase pre_src_compile - vecho ">>> Compiling source in $PWD ..." - ebuild_phase src_compile + __vecho ">>> Compiling source in $PWD ..." + __ebuild_phase src_compile >> "$PORTAGE_BUILDDIR/.compiled" || \ die "Failed to create $PORTAGE_BUILDDIR/.compiled" - vecho ">>> Source compiled." + __vecho ">>> Source compiled." - ebuild_phase post_src_compile + __ebuild_phase post_src_compile trap - SIGINT SIGQUIT } -dyn_test() { +__dyn_test() { if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then - vecho ">>> It appears that ${PN} has already been tested; skipping." - vecho ">>> Remove '${PORTAGE_BUILDDIR}/.tested' to force test." + __vecho ">>> It appears that ${PN} has already been tested; skipping." + __vecho ">>> Remove '${PORTAGE_BUILDDIR}/.tested' to force test." return fi - if [ "${EBUILD_FORCE_TEST}" == "1" ] ; then - # If USE came from ${T}/environment then it might not have USE=test - # like it's supposed to here. - ! has test ${USE} && export USE="${USE} test" - fi - - trap "abort_test" SIGINT SIGQUIT + trap "__abort_test" SIGINT SIGQUIT if [ -d "${S}" ]; then cd "${S}" else cd "${WORKDIR}" fi - if ! has test $FEATURES && [ "${EBUILD_FORCE_TEST}" != "1" ]; then - vecho ">>> Test phase [not enabled]: ${CATEGORY}/${PF}" - elif has test $RESTRICT; then + if has test ${RESTRICT} ; then einfo "Skipping make test/check due to ebuild restriction." - vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" + __vecho ">>> Test phase [disabled because of RESTRICT=test]: ${CATEGORY}/${PF}" + + # If ${EBUILD_FORCE_TEST} == 1 and FEATURES came from ${T}/environment + # then it might not have FEATURES=test like it's supposed to here. + elif [[ ${EBUILD_FORCE_TEST} != 1 ]] && ! has test ${FEATURES} ; then + __vecho ">>> Test phase [not enabled]: ${CATEGORY}/${PF}" else + # If ${EBUILD_FORCE_TEST} == 1 and USE came from ${T}/environment + # then it might not have USE=test like it's supposed to here. + if [[ ${EBUILD_FORCE_TEST} == 1 && test =~ ${PORTAGE_IUSE} ]] && \ + ! has test ${USE} ; then + export USE="${USE} test" + fi + local save_sp=${SANDBOX_PREDICT} addpredict / - ebuild_phase pre_src_test - ebuild_phase src_test + __ebuild_phase pre_src_test + __ebuild_phase src_test >> "$PORTAGE_BUILDDIR/.tested" || \ die "Failed to create $PORTAGE_BUILDDIR/.tested" - ebuild_phase post_src_test + __ebuild_phase post_src_test SANDBOX_PREDICT=${save_sp} fi trap - SIGINT SIGQUIT } -dyn_install() { +__dyn_install() { [ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset" if has noauto $FEATURES ; then rm -f "${PORTAGE_BUILDDIR}/.installed" elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then - vecho ">>> It appears that '${PF}' is already installed; skipping." - vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install." + __vecho ">>> It appears that '${PF}' is already installed; skipping." + __vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install." return 0 fi - trap "abort_install" SIGINT SIGQUIT - ebuild_phase pre_src_install + trap "__abort_install" SIGINT SIGQUIT + __ebuild_phase pre_src_install - _x=${ED} - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) _x=${D} ;; esac + if ___eapi_has_prefix_variables; then + _x=${ED} + else + _x=${D} + fi rm -rf "${D}" mkdir -p "${_x}" unset _x if [[ -d $S ]] ; then cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then + elif ___eapi_has_S_WORKDIR_fallback; then cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to install; then + elif [[ -z ${A} ]] && ! __has_phase_defined_up_to install; then cd "${WORKDIR}" else die "The source directory '${S}' doesn't exist" fi - vecho - vecho ">>> Install ${PF} into ${D} category ${CATEGORY}" + __vecho + __vecho ">>> Install ${PF} into ${D} category ${CATEGORY}" #our custom version of libtool uses $S and $D to fix #invalid paths in .la files export S D @@ -541,12 +544,12 @@ dyn_install() { export _E_EXEDESTTREE_="" export _E_DOCDESTTREE_="" - ebuild_phase src_install + __ebuild_phase src_install >> "$PORTAGE_BUILDDIR/.installed" || \ die "Failed to create $PORTAGE_BUILDDIR/.installed" - vecho ">>> Completed installing ${PF} into ${D}" - vecho - ebuild_phase post_src_install + __vecho ">>> Completed installing ${PF} into ${D}" + __vecho + __ebuild_phase post_src_install cd "${PORTAGE_BUILDDIR}"/build-info set -f @@ -560,10 +563,15 @@ dyn_install() { if [[ $CATEGORY != virtual ]] ; then for f in ASFLAGS CBUILD CC CFLAGS CHOST CTARGET CXX \ CXXFLAGS EXTRA_ECONF EXTRA_EINSTALL EXTRA_MAKE \ - LDFLAGS LIBCFLAGS LIBCXXFLAGS ; do + LDFLAGS LIBCFLAGS LIBCXXFLAGS QA_CONFIGURE_OPTIONS \ + QA_DESKTOP_FILE ; do x=$(echo -n ${!f}) [[ -n $x ]] && echo "$x" > $f done + # whitespace preserved + for f in QA_AM_MAINTAINER_MODE ; do + [[ -n ${!f} ]] && echo "${!f}" > $f + done fi echo "${USE}" > USE echo "${EAPI:-0}" > EAPI @@ -571,24 +579,22 @@ dyn_install() { # Save EPREFIX, since it makes it easy to use chpathtool to # adjust the content of a binary package so that it will # work in a different EPREFIX from the one is was built for. - case "${EAPI:-0}" in - 0|1|2) - [[ " ${FEATURES} " == *" force-prefix "* ]] && \ - [ -n "${EPREFIX}" ] && echo "${EPREFIX}" > EPREFIX - ;; - *) - [ -n "${EPREFIX}" ] && echo "${EPREFIX}" > EPREFIX - ;; - esac + if ___eapi_has_prefix_variables && [[ -n ${EPREFIX} ]]; then + echo "${EPREFIX}" > EPREFIX + fi set +f # local variables can leak into the saved environment. unset f - save_ebuild_env --exclude-init-phases | filter_readonly_variables \ - --filter-path --filter-sandbox --allow-extra-vars > environment - assert "save_ebuild_env failed" + # Use safe cwd, avoiding unsafe import for bug #469338. + cd "${PORTAGE_PYM_PATH}" + __save_ebuild_env --exclude-init-phases | __filter_readonly_variables \ + --filter-path --filter-sandbox --allow-extra-vars > \ + "${PORTAGE_BUILDDIR}"/build-info/environment + assert "__save_ebuild_env failed" + cd "${PORTAGE_BUILDDIR}"/build-info || die ${PORTAGE_BZIP2_COMMAND} -f9 environment @@ -601,15 +607,7 @@ dyn_install() { trap - SIGINT SIGQUIT } -dyn_preinst() { - if [ -z "${D}" ]; then - eerror "${FUNCNAME}: D is unset" - return 1 - fi - ebuild_phase_with_hooks pkg_preinst -} - -dyn_help() { +__dyn_help() { echo echo "Portage" echo "Copyright 1999-2010 Gentoo Foundation" @@ -625,6 +623,7 @@ dyn_help() { echo " pretend : execute package specific pretend actions" echo " setup : execute package specific setup actions" echo " fetch : download source archive(s) and patches" + echo " nofetch : display special fetch instructions" echo " digest : create a manifest file for the package" echo " manifest : create a manifest file for the package" echo " unpack : unpack sources (auto-dependencies if needed)" @@ -672,19 +671,18 @@ dyn_help() { echo } -# @FUNCTION: _ebuild_arg_to_phase +# @FUNCTION: __ebuild_arg_to_phase # @DESCRIPTION: # Translate a known ebuild(1) argument into the precise # name of it's corresponding ebuild phase. -_ebuild_arg_to_phase() { - [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" - local eapi=$1 - local arg=$2 +__ebuild_arg_to_phase() { + [ $# -ne 1 ] && die "expected exactly 1 arg, got $#: $*" + local arg=$1 local phase_func="" case "$arg" in pretend) - ! has $eapi 0 1 2 3 3_pre2 && \ + ___eapi_has_pkg_pretend && \ phase_func=pkg_pretend ;; setup) @@ -697,11 +695,11 @@ _ebuild_arg_to_phase() { phase_func=src_unpack ;; prepare) - ! has $eapi 0 1 && \ + ___eapi_has_src_prepare && \ phase_func=src_prepare ;; configure) - ! has $eapi 0 1 && \ + ___eapi_has_src_configure && \ phase_func=src_configure ;; compile) @@ -732,7 +730,7 @@ _ebuild_arg_to_phase() { return 0 } -_ebuild_phase_funcs() { +__ebuild_phase_funcs() { [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" local eapi=$1 local phase_func=$2 @@ -742,20 +740,20 @@ _ebuild_phase_funcs() { for x in pkg_nofetch src_unpack src_test ; do declare -F $x >/dev/null || \ - eval "$x() { _eapi0_$x \"\$@\" ; }" + eval "$x() { __eapi0_$x \"\$@\" ; }" done - case $eapi in + case "$eapi" in 0|1) if ! declare -F src_compile >/dev/null ; then - case $eapi in + case "$eapi" in 0) - src_compile() { _eapi0_src_compile "$@" ; } + src_compile() { __eapi0_src_compile "$@" ; } ;; *) - src_compile() { _eapi1_src_compile "$@" ; } + src_compile() { __eapi1_src_compile "$@" ; } ;; esac fi @@ -775,35 +773,35 @@ _ebuild_phase_funcs() { *) declare -F src_configure >/dev/null || \ - src_configure() { _eapi2_src_configure "$@" ; } + src_configure() { __eapi2_src_configure "$@" ; } declare -F src_compile >/dev/null || \ - src_compile() { _eapi2_src_compile "$@" ; } + src_compile() { __eapi2_src_compile "$@" ; } - has $eapi 2 3 3_pre2 || declare -F src_install >/dev/null || \ - src_install() { _eapi4_src_install "$@" ; } + has $eapi 2 3 || declare -F src_install >/dev/null || \ + src_install() { __eapi4_src_install "$@" ; } if has $phase_func $default_phases ; then - _eapi2_pkg_nofetch () { _eapi0_pkg_nofetch "$@" ; } - _eapi2_src_unpack () { _eapi0_src_unpack "$@" ; } - _eapi2_src_prepare () { true ; } - _eapi2_src_test () { _eapi0_src_test "$@" ; } - _eapi2_src_install () { die "$FUNCNAME is not supported" ; } + __eapi2_pkg_nofetch () { __eapi0_pkg_nofetch "$@" ; } + __eapi2_src_unpack () { __eapi0_src_unpack "$@" ; } + __eapi2_src_prepare () { true ; } + __eapi2_src_test () { __eapi0_src_test "$@" ; } + __eapi2_src_install () { die "$FUNCNAME is not supported" ; } for x in $default_phases ; do - eval "default_$x() { _eapi2_$x \"\$@\" ; }" + eval "default_$x() { __eapi2_$x \"\$@\" ; }" done - eval "default() { _eapi2_$phase_func \"\$@\" ; }" + eval "default() { __eapi2_$phase_func \"\$@\" ; }" - case $eapi in + case "$eapi" in 2|3) ;; *) - eval "default_src_install() { _eapi4_src_install \"\$@\" ; }" + eval "default_src_install() { __eapi4_src_install \"\$@\" ; }" [[ $phase_func = src_install ]] && \ - eval "default() { _eapi4_$phase_func \"\$@\" ; }" + eval "default() { __eapi4_$phase_func \"\$@\" ; }" ;; esac @@ -825,14 +823,14 @@ _ebuild_phase_funcs() { esac } -ebuild_main() { +__ebuild_main() { # Subshell/helper die support (must export for the die helper). # Since this function is typically executed in a subshell, # setup EBUILD_MASTER_PID to refer to the current $BASHPID, # which seems to give the best results when further # nested subshells call die. - export EBUILD_MASTER_PID=$BASHPID + export EBUILD_MASTER_PID=${BASHPID:-$(__bashpid)} trap 'exit 1' SIGTERM #a reasonable default for $S @@ -861,37 +859,39 @@ ebuild_main() { # respect FEATURES="-ccache". has ccache $FEATURES || export CCACHE_DISABLE=1 - local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") - [[ -n $phase_func ]] && _ebuild_phase_funcs "$EAPI" "$phase_func" + local phase_func=$(__ebuild_arg_to_phase "$EBUILD_PHASE") + [[ -n $phase_func ]] && __ebuild_phase_funcs "$EAPI" "$phase_func" unset phase_func - source_all_bashrcs + __source_all_bashrcs case ${1} in nofetch) - ebuild_phase_with_hooks pkg_nofetch + __ebuild_phase_with_hooks pkg_nofetch ;; - prerm|postrm|postinst|config|info) + prerm|postrm|preinst|postinst|config|info) if has "${1}" config info && \ ! declare -F "pkg_${1}" >/dev/null ; then ewarn "pkg_${1}() is not defined: '${EBUILD##*/}'" fi export SANDBOX_ON="0" if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - ebuild_phase_with_hooks pkg_${1} + __ebuild_phase_with_hooks pkg_${1} else set -x - ebuild_phase_with_hooks pkg_${1} + __ebuild_phase_with_hooks pkg_${1} set +x fi - if [[ $EBUILD_PHASE == postinst ]] && [[ -n $PORTAGE_UPDATE_ENV ]]; then + if [[ -n $PORTAGE_UPDATE_ENV ]] ; then # Update environment.bz2 in case installation phases # need to pass some variables to uninstallation phases. - save_ebuild_env --exclude-init-phases | \ - filter_readonly_variables --filter-path \ + # Use safe cwd, avoiding unsafe import for bug #469338. + cd "${PORTAGE_PYM_PATH}" + __save_ebuild_env --exclude-init-phases | \ + __filter_readonly_variables --filter-path \ --filter-sandbox --allow-extra-vars \ | ${PORTAGE_BZIP2_COMMAND} -c -f9 > "$PORTAGE_UPDATE_ENV" - assert "save_ebuild_env failed" + assert "__save_ebuild_env failed" fi ;; unpack|prepare|configure|compile|test|clean|install) @@ -917,7 +917,7 @@ ebuild_main() { x=LIBDIR_$ABI [ -z "$PKG_CONFIG_PATH" -a -n "$ABI" -a -n "${!x}" ] && \ - export PKG_CONFIG_PATH=/usr/${!x}/pkgconfig + export PKG_CONFIG_PATH=${EPREFIX}/usr/${!x}/pkgconfig if has noauto $FEATURES && \ [[ ! -f $PORTAGE_BUILDDIR/.unpacked ]] ; then @@ -952,24 +952,24 @@ ebuild_main() { esac if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - dyn_${1} + __dyn_${1} else set -x - dyn_${1} + __dyn_${1} set +x fi export SANDBOX_ON="0" ;; - help|pretend|setup|preinst) + help|pretend|setup) #pkg_setup needs to be out of the sandbox for tmp file creation; #for example, awking and piping a file in /tmp requires a temp file to be created #in /etc. If pkg_setup is in the sandbox, both our lilo and apache ebuilds break. export SANDBOX_ON="0" if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - dyn_${1} + __dyn_${1} else set -x - dyn_${1} + __dyn_${1} set +x fi ;; @@ -979,7 +979,7 @@ ebuild_main() { export SANDBOX_ON="1" echo "Unrecognized arg '${1}'" echo - dyn_help + __dyn_help exit 1 ;; esac @@ -987,11 +987,13 @@ ebuild_main() { # Save the env only for relevant phases. if ! has "${1}" clean help info nofetch ; then umask 002 - save_ebuild_env | filter_readonly_variables \ + # Use safe cwd, avoiding unsafe import for bug #469338. + cd "${PORTAGE_PYM_PATH}" + __save_ebuild_env | __filter_readonly_variables \ --filter-features > "$T/environment" - assert "save_ebuild_env failed" - chown portage:portage "$T/environment" &>/dev/null - chmod g+w "$T/environment" &>/dev/null + assert "__save_ebuild_env failed" + chgrp "${PORTAGE_GRPNAME:-portage}" "$T/environment" + chmod g+w "$T/environment" fi [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE" if [[ -n $PORTAGE_IPC_DAEMON ]] ; then diff --git a/portage_with_autodep/bin/phase-helpers.sh b/portage_with_autodep/bin/phase-helpers.sh index 946520b..412decb 100755 --- a/portage_with_autodep/bin/phase-helpers.sh +++ b/portage_with_autodep/bin/phase-helpers.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 export DESTTREE=/usr @@ -11,6 +11,8 @@ export EXEOPTIONS="-m0755" export LIBOPTIONS="-m0644" export DIROPTIONS="-m0755" export MOPREFIX=${PN} +# Do not compress files which are smaller than this (in bytes). #169260 +export PORTAGE_DOCOMPRESS_SIZE_LIMIT="128" declare -a PORTAGE_DOCOMPRESS=( /usr/share/{doc,info,man} ) declare -a PORTAGE_DOCOMPRESS_SKIP=( /usr/share/doc/${PF}/html ) @@ -19,13 +21,14 @@ into() { export DESTTREE="" else export DESTTREE=$1 - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi if [ ! -d "${ED}${DESTTREE}" ]; then install -d "${ED}${DESTTREE}" local ret=$? if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" + __helpers_die "${FUNCNAME[0]} failed" return $ret fi fi @@ -37,13 +40,14 @@ insinto() { export INSDESTTREE="" else export INSDESTTREE=$1 - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi if [ ! -d "${ED}${INSDESTTREE}" ]; then install -d "${ED}${INSDESTTREE}" local ret=$? if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" + __helpers_die "${FUNCNAME[0]} failed" return $ret fi fi @@ -55,13 +59,14 @@ exeinto() { export _E_EXEDESTTREE_="" else export _E_EXEDESTTREE_="$1" - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi if [ ! -d "${ED}${_E_EXEDESTTREE_}" ]; then install -d "${ED}${_E_EXEDESTTREE_}" local ret=$? if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" + __helpers_die "${FUNCNAME[0]} failed" return $ret fi fi @@ -73,13 +78,14 @@ docinto() { export _E_DOCDESTTREE_="" else export _E_DOCDESTTREE_="$1" - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi if [ ! -d "${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" ]; then install -d "${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" local ret=$? if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" + __helpers_die "${FUNCNAME[0]} failed" return $ret fi fi @@ -112,13 +118,13 @@ libopts() { } docompress() { - has "${EAPI}" 0 1 2 3 && die "'docompress' not supported in this EAPI" + ___eapi_has_docompress || die "'docompress' not supported in this EAPI" local f g if [[ $1 = "-x" ]]; then shift for f; do - f=$(strip_duplicate_slashes "${f}"); f=${f%/} + f=$(__strip_duplicate_slashes "${f}"); f=${f%/} [[ ${f:0:1} = / ]] || f="/${f}" for g in "${PORTAGE_DOCOMPRESS_SKIP[@]}"; do [[ ${f} = "${g}" ]] && continue 2 @@ -127,7 +133,7 @@ docompress() { done else for f; do - f=$(strip_duplicate_slashes "${f}"); f=${f%/} + f=$(__strip_duplicate_slashes "${f}"); f=${f%/} [[ ${f:0:1} = / ]] || f="/${f}" for g in "${PORTAGE_DOCOMPRESS[@]}"; do [[ ${f} = "${g}" ]] && continue 2 @@ -137,29 +143,6 @@ docompress() { fi } -# adds ".keep" files so that dirs aren't auto-cleaned -keepdir() { - dodir "$@" - local x - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac - if [ "$1" == "-R" ] || [ "$1" == "-r" ]; then - shift - find "$@" -type d -printf "${ED}%p/.keep_${CATEGORY}_${PN}-${SLOT}\n" \ - | tr "\n" "\0" | \ - while read -r -d $'\0' ; do - >> "$REPLY" || \ - die "Failed to recursively create .keep files" - done - else - for x in "$@"; do - >> "${ED}${x}/.keep_${CATEGORY}_${PN}-${SLOT}" || \ - die "Failed to create .keep in ${ED}${x}" - done - fi -} - - useq() { has $EBUILD_PHASE prerm postrm || eqawarn \ "QA Notice: The 'useq' function is deprecated (replaced by 'use')" @@ -174,6 +157,17 @@ usev() { return 1 } +if ___eapi_has_usex; then + usex() { + if use "$1"; then + echo "${2-yes}$4" + else + echo "${3-no}$5" + fi + return 0 + } +fi + use() { local u=$1 local found=0 @@ -194,18 +188,31 @@ use() { #fi true - # Make sure we have this USE flag in IUSE - elif [[ -n $PORTAGE_IUSE && -n $EBUILD_PHASE ]] ; then - [[ $u =~ $PORTAGE_IUSE ]] || \ + # Make sure we have this USE flag in IUSE, but exempt binary + # packages for API consumers like Entropy which do not require + # a full profile with IUSE_IMPLICIT and stuff (see bug #456830). + elif [[ -n $PORTAGE_IUSE && -n $EBUILD_PHASE && + -n $PORTAGE_INTERNAL_CALLER ]] ; then + if [[ ! $u =~ $PORTAGE_IUSE ]] ; then + if [[ ! ${EAPI} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] ; then + # This is only strict starting with EAPI 5, since implicit IUSE + # is not well defined for earlier EAPIs (see bug #449708). + die "USE Flag '${u}' not in IUSE for ${CATEGORY}/${PF}" + fi eqawarn "QA Notice: USE Flag '${u}' not" \ "in IUSE for ${CATEGORY}/${PF}" + fi fi + local IFS=$' \t\n' prev_shopts=$- ret + set -f if has ${u} ${USE} ; then - return ${found} + ret=${found} else - return $((!found)) + ret=$((!found)) fi + [[ ${prev_shopts} == *f* ]] || set +f + return ${ret} } use_with() { @@ -215,7 +222,7 @@ use_with() { return 1 fi - if ! has "${EAPI:-0}" 0 1 2 3 ; then + if ___eapi_use_enable_and_use_with_support_empty_third_argument; then local UW_SUFFIX=${3+=$3} else local UW_SUFFIX=${3:+=$3} @@ -237,7 +244,7 @@ use_enable() { return 1 fi - if ! has "${EAPI:-0}" 0 1 2 3 ; then + if ___eapi_use_enable_and_use_with_support_empty_third_argument; then local UE_SUFFIX=${3+=$3} else local UE_SUFFIX=${3:+=$3} @@ -255,15 +262,19 @@ use_enable() { unpack() { local srcdir local x - local y + local y y_insensitive + local suffix suffix_insensitive local myfail local eapi=${EAPI:-0} [ -z "$*" ] && die "Nothing passed to the 'unpack' command" for x in "$@"; do - vecho ">>> Unpacking ${x} to ${PWD}" + __vecho ">>> Unpacking ${x} to ${PWD}" + suffix=${x##*.} + suffix_insensitive=$(LC_ALL=C tr "[:upper:]" "[:lower:]" <<< "${suffix}") y=${x%.*} y=${y##*.} + y_insensitive=$(LC_ALL=C tr "[:upper:]" "[:lower:]" <<< "${y}") if [[ ${x} == "./"* ]] ; then srcdir="" @@ -276,10 +287,16 @@ unpack() { fi [[ ! -s ${srcdir}${x} ]] && die "${x} does not exist" - _unpack_tar() { - if [ "${y}" == "tar" ]; then + __unpack_tar() { + if [[ ${y_insensitive} == tar ]] ; then + if ___eapi_unpack_is_case_sensitive && \ + [[ tar != ${y} ]] ; then + eqawarn "QA Notice: unpack called with" \ + "secondary suffix '${y}' which is unofficially" \ + "supported with EAPI '${EAPI}'. Instead use 'tar'." + fi $1 -c -- "$srcdir$x" | tar xof - - assert_sigpipe_ok "$myfail" + __assert_sigpipe_ok "$myfail" else local cwd_dest=${x##*/} cwd_dest=${cwd_dest%.*} @@ -288,30 +305,67 @@ unpack() { } myfail="failure unpacking ${x}" - case "${x##*.}" in + case "${suffix_insensitive}" in tar) + if ___eapi_unpack_is_case_sensitive && \ + [[ tar != ${suffix} ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'tar'." + fi tar xof "$srcdir$x" || die "$myfail" ;; tgz) + if ___eapi_unpack_is_case_sensitive && \ + [[ tgz != ${suffix} ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'tgz'." + fi tar xozf "$srcdir$x" || die "$myfail" ;; tbz|tbz2) + if ___eapi_unpack_is_case_sensitive && \ + [[ " tbz tbz2 " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'tbz' or 'tbz2'." + fi ${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- "$srcdir$x" | tar xof - - assert_sigpipe_ok "$myfail" + __assert_sigpipe_ok "$myfail" ;; - ZIP|zip|jar) + zip|jar) + if ___eapi_unpack_is_case_sensitive && \ + [[ " ZIP zip jar " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'." \ + "Instead use 'ZIP', 'zip', or 'jar'." + fi # unzip will interactively prompt under some error conditions, # as reported in bug #336285 ( set +x ; while true ; do echo n || break ; done ) | \ unzip -qo "${srcdir}${x}" || die "$myfail" ;; - gz|Z|z) - _unpack_tar "gzip -d" + gz|z) + if ___eapi_unpack_is_case_sensitive && \ + [[ " gz z Z " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'gz', 'z', or 'Z'." + fi + __unpack_tar "gzip -d" ;; bz2|bz) - _unpack_tar "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}" + if ___eapi_unpack_is_case_sensitive && \ + [[ " bz bz2 " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'bz' or 'bz2'." + fi + __unpack_tar "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}" ;; - 7Z|7z) + 7z) local my_output my_output="$(7z x -y "${srcdir}${x}")" if [ $? -ne 0 ]; then @@ -319,16 +373,41 @@ unpack() { die "$myfail" fi ;; - RAR|rar) + rar) + if ___eapi_unpack_is_case_sensitive && \ + [[ " rar RAR " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'rar' or 'RAR'." + fi unrar x -idq -o+ "${srcdir}${x}" || die "$myfail" ;; - LHa|LHA|lha|lzh) + lha|lzh) + if ___eapi_unpack_is_case_sensitive && \ + [[ " LHA LHa lha lzh " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'." \ + "Instead use 'LHA', 'LHa', 'lha', or 'lzh'." + fi lha xfq "${srcdir}${x}" || die "$myfail" ;; a) + if ___eapi_unpack_is_case_sensitive && \ + [[ " a " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'a'." + fi ar x "${srcdir}${x}" || die "$myfail" ;; deb) + if ___eapi_unpack_is_case_sensitive && \ + [[ " deb " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'deb'." + fi # Unpacking .deb archives can not always be done with # `ar`. For instance on AIX this doesn't work out. If # we have `deb2targz` installed, prefer it over `ar` for @@ -356,17 +435,29 @@ unpack() { fi ;; lzma) - _unpack_tar "lzma -d" + if ___eapi_unpack_is_case_sensitive && \ + [[ " lzma " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'lzma'." + fi + __unpack_tar "lzma -d" ;; xz) - if has $eapi 0 1 2 ; then - vecho "unpack ${x}: file format not recognized. Ignoring." + if ___eapi_unpack_is_case_sensitive && \ + [[ " xz " != *" ${suffix} "* ]] ; then + eqawarn "QA Notice: unpack called with" \ + "suffix '${suffix}' which is unofficially supported" \ + "with EAPI '${EAPI}'. Instead use 'xz'." + fi + if ___eapi_unpack_supports_xz; then + __unpack_tar "xz -d" else - _unpack_tar "xz -d" + __vecho "unpack ${x}: file format not recognized. Ignoring." fi ;; *) - vecho "unpack ${x}: file format not recognized. Ignoring." + __vecho "unpack ${x}: file format not recognized. Ignoring." ;; esac done @@ -378,22 +469,24 @@ unpack() { econf() { local x + local pid=${BASHPID:-$(__bashpid)} - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local EPREFIX= ;; esac + if ! ___eapi_has_prefix_variables; then + local EPREFIX= + fi - _hasg() { + __hasg() { local x s=$1 shift for x ; do [[ ${x} == ${s} ]] && echo "${x}" && return 0 ; done return 1 } - _hasgq() { _hasg "$@" >/dev/null ; } + __hasgq() { __hasg "$@" >/dev/null ; } - local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") + local phase_func=$(__ebuild_arg_to_phase "$EBUILD_PHASE") if [[ -n $phase_func ]] ; then - if has "$EAPI" 0 1 ; then + if ! ___eapi_has_src_configure; then [[ $phase_func != src_compile ]] && \ eqawarn "QA Notice: econf called in" \ "$phase_func instead of src_compile" @@ -408,23 +501,44 @@ econf() { if [ -x "${ECONF_SOURCE}/configure" ]; then if [[ -n $CONFIG_SHELL && \ "$(head -n1 "$ECONF_SOURCE/configure")" =~ ^'#!'[[:space:]]*/bin/sh([[:space:]]|$) ]] ; then - sed -e "1s:^#![[:space:]]*/bin/sh:#!$CONFIG_SHELL:" -i "$ECONF_SOURCE/configure" || \ - die "Substition of shebang in '$ECONF_SOURCE/configure' failed" + # preserve timestamp, see bug #440304 + touch -r "${ECONF_SOURCE}/configure" "${ECONF_SOURCE}/configure._portage_tmp_.${pid}" || die + sed -i \ + -e "1s:^#![[:space:]]*/bin/sh:#!$CONFIG_SHELL:" \ + "${ECONF_SOURCE}/configure" \ + || die "Substition of shebang in '${ECONF_SOURCE}/configure' failed" + touch -r "${ECONF_SOURCE}/configure._portage_tmp_.${pid}" "${ECONF_SOURCE}/configure" || die + rm -f "${ECONF_SOURCE}/configure._portage_tmp_.${pid}" fi if [ -e "${EPREFIX}"/usr/share/gnuconfig/ ]; then find "${WORKDIR}" -type f '(' \ -name config.guess -o -name config.sub ')' -print0 | \ while read -r -d $'\0' x ; do - vecho " * econf: updating ${x/${WORKDIR}\/} with ${EPREFIX}/usr/share/gnuconfig/${x##*/}" - cp -f "${EPREFIX}"/usr/share/gnuconfig/"${x##*/}" "${x}" + __vecho " * econf: updating ${x/${WORKDIR}\/} with ${EPREFIX}/usr/share/gnuconfig/${x##*/}" + # Make sure we do this atomically incase we're run in parallel. #487478 + cp -f "${EPREFIX}"/usr/share/gnuconfig/"${x##*/}" "${x}.${pid}" + mv -f "${x}.${pid}" "${x}" done fi - # EAPI=4 adds --disable-dependency-tracking to econf - if ! has "$EAPI" 0 1 2 3 3_pre2 && \ - "${ECONF_SOURCE}/configure" --help 2>/dev/null | \ - grep -q disable-dependency-tracking ; then - set -- --disable-dependency-tracking "$@" + if ___eapi_econf_passes_--disable-dependency-tracking || ___eapi_econf_passes_--disable-silent-rules; then + local conf_help=$("${ECONF_SOURCE}/configure" --help 2>/dev/null) + + if ___eapi_econf_passes_--disable-dependency-tracking; then + case "${conf_help}" in + *--disable-dependency-tracking*) + set -- --disable-dependency-tracking "$@" + ;; + esac + fi + + if ___eapi_econf_passes_--disable-silent-rules; then + case "${conf_help}" in + *--disable-silent-rules*) + set -- --disable-silent-rules "$@" + ;; + esac + fi fi # if the profile defines a location to install libs to aside from default, pass it on. @@ -433,16 +547,19 @@ econf() { if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then CONF_LIBDIR=${!LIBDIR_VAR} fi - if [[ -n ${CONF_LIBDIR} ]] && ! _hasgq --libdir=\* "$@" ; then - export CONF_PREFIX=$(_hasg --exec-prefix=\* "$@") - [[ -z ${CONF_PREFIX} ]] && CONF_PREFIX=$(_hasg --prefix=\* "$@") + if [[ -n ${CONF_LIBDIR} ]] && ! __hasgq --libdir=\* "$@" ; then + export CONF_PREFIX=$(__hasg --exec-prefix=\* "$@") + [[ -z ${CONF_PREFIX} ]] && CONF_PREFIX=$(__hasg --prefix=\* "$@") : ${CONF_PREFIX:=${EPREFIX}/usr} CONF_PREFIX=${CONF_PREFIX#*=} [[ ${CONF_PREFIX} != /* ]] && CONF_PREFIX="/${CONF_PREFIX}" [[ ${CONF_LIBDIR} != /* ]] && CONF_LIBDIR="/${CONF_LIBDIR}" - set -- --libdir="$(strip_duplicate_slashes ${CONF_PREFIX}${CONF_LIBDIR})" "$@" + set -- --libdir="$(__strip_duplicate_slashes "${CONF_PREFIX}${CONF_LIBDIR}")" "$@" fi + # Handle arguments containing quoted whitespace (see bug #457136). + eval "local -a EXTRA_ECONF=(${EXTRA_ECONF})" + set -- \ --prefix="${EPREFIX}"/usr \ ${CBUILD:+--build=${CBUILD}} \ @@ -454,8 +571,8 @@ econf() { --sysconfdir="${EPREFIX}"/etc \ --localstatedir="${EPREFIX}"/var/lib \ "$@" \ - ${EXTRA_ECONF} - vecho "${ECONF_SOURCE}/configure" "$@" + "${EXTRA_ECONF[@]}" + __vecho "${ECONF_SOURCE}/configure" "$@" if ! "${ECONF_SOURCE}/configure" "$@" ; then @@ -476,8 +593,9 @@ econf() { einstall() { # CONF_PREFIX is only set if they didn't pass in libdir above. local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}" - [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if ! ___eapi_has_prefix_variables; then + local ED=${D} + fi LIBDIR_VAR="LIBDIR_${ABI}" if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then CONF_LIBDIR="${!LIBDIR_VAR}" @@ -485,7 +603,7 @@ einstall() { unset LIBDIR_VAR if [ -n "${CONF_LIBDIR}" ] && [ "${CONF_PREFIX:+set}" = set ]; then EI_DESTLIBDIR="${D}/${CONF_PREFIX}/${CONF_LIBDIR}" - EI_DESTLIBDIR="$(strip_duplicate_slashes ${EI_DESTLIBDIR})" + EI_DESTLIBDIR="$(__strip_duplicate_slashes "${EI_DESTLIBDIR}")" LOCAL_EXTRA_EINSTALL="libdir=${EI_DESTLIBDIR} ${LOCAL_EXTRA_EINSTALL}" unset EI_DESTLIBDIR fi @@ -516,7 +634,7 @@ einstall() { fi } -_eapi0_pkg_nofetch() { +__eapi0_pkg_nofetch() { [ -z "${SRC_URI}" ] && return elog "The following are listed in SRC_URI for ${PN}:" @@ -526,55 +644,59 @@ _eapi0_pkg_nofetch() { done } -_eapi0_src_unpack() { +__eapi0_src_unpack() { [[ -n ${A} ]] && unpack ${A} } -_eapi0_src_compile() { +__eapi0_src_compile() { if [ -x ./configure ] ; then econf fi - _eapi2_src_compile + __eapi2_src_compile } -_eapi0_src_test() { +__eapi0_src_test() { # Since we don't want emake's automatic die # support (EAPI 4 and later), and we also don't # want the warning messages that it produces if # we call it in 'nonfatal' mode, we use emake_cmd # to emulate the desired parts of emake behavior. local emake_cmd="${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE}" - if $emake_cmd -j1 check -n &> /dev/null; then - vecho ">>> Test phase [check]: ${CATEGORY}/${PF}" - $emake_cmd -j1 check || \ + local internal_opts= + if ___eapi_default_src_test_disables_parallel_jobs; then + internal_opts+=" -j1" + fi + if $emake_cmd ${internal_opts} check -n &> /dev/null; then + __vecho ">>> Test phase [check]: ${CATEGORY}/${PF}" + $emake_cmd ${internal_opts} check || \ die "Make check failed. See above for details." - elif $emake_cmd -j1 test -n &> /dev/null; then - vecho ">>> Test phase [test]: ${CATEGORY}/${PF}" - $emake_cmd -j1 test || \ + elif $emake_cmd ${internal_opts} test -n &> /dev/null; then + __vecho ">>> Test phase [test]: ${CATEGORY}/${PF}" + $emake_cmd ${internal_opts} test || \ die "Make test failed. See above for details." else - vecho ">>> Test phase [none]: ${CATEGORY}/${PF}" + __vecho ">>> Test phase [none]: ${CATEGORY}/${PF}" fi } -_eapi1_src_compile() { - _eapi2_src_configure - _eapi2_src_compile +__eapi1_src_compile() { + __eapi2_src_configure + __eapi2_src_compile } -_eapi2_src_configure() { +__eapi2_src_configure() { if [[ -x ${ECONF_SOURCE:-.}/configure ]] ; then econf fi } -_eapi2_src_compile() { +__eapi2_src_compile() { if [ -f Makefile ] || [ -f GNUmakefile ] || [ -f makefile ]; then emake || die "emake failed" fi } -_eapi4_src_install() { +__eapi4_src_install() { if [[ -f Makefile || -f GNUmakefile || -f makefile ]] ; then emake DESTDIR="${D}" install fi @@ -593,71 +715,285 @@ _eapi4_src_install() { } # @FUNCTION: has_version -# @USAGE: <DEPEND ATOM> +# @USAGE: [--host-root] <DEPEND ATOM> # @DESCRIPTION: # Return true if given package is installed. Otherwise return false. # Callers may override the ROOT variable in order to match packages from an # alternative ROOT. has_version() { - local eroot - case "$EAPI" in - 0|1|2) - [[ " ${FEATURES} " == *" force-prefix "* ]] && \ - eroot=${ROOT%/}${EPREFIX}/ || eroot=${ROOT} - ;; - *) - eroot=${ROOT%/}${EPREFIX}/ - ;; - esac + local atom eroot host_root=false root=${ROOT} + if [[ $1 == --host-root ]] ; then + host_root=true + shift + fi + atom=$1 + shift + [ $# -gt 0 ] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if ${host_root} ; then + if ! ___eapi_best_version_and_has_version_support_--host-root; then + die "${FUNCNAME[0]}: option --host-root is not supported with EAPI ${EAPI}" + fi + root=/ + fi + + if ___eapi_has_prefix_variables; then + # [[ ${root} == / ]] would be ambiguous here, + # since both prefixes can share root=/ while + # having different EPREFIX offsets. + if ${host_root} ; then + eroot=${root%/}${PORTAGE_OVERRIDE_EPREFIX}/ + else + eroot=${root%/}${EPREFIX}/ + fi + else + eroot=${root} + fi if [[ -n $PORTAGE_IPC_DAEMON ]] ; then - "$PORTAGE_BIN_PATH"/ebuild-ipc has_version "${eroot}" "$1" + "$PORTAGE_BIN_PATH"/ebuild-ipc has_version "${eroot}" "${atom}" else - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" has_version "${eroot}" "$1" + "${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" has_version "${eroot}" "${atom}" fi local retval=$? case "${retval}" in 0|1) return ${retval} ;; + 2) + die "${FUNCNAME[0]}: invalid atom: ${atom}" + ;; *) - die "unexpected portageq exit code: ${retval}" + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi ;; esac } # @FUNCTION: best_version -# @USAGE: <DEPEND ATOM> +# @USAGE: [--host-root] <DEPEND ATOM> # @DESCRIPTION: # Returns the best/most-current match. # Callers may override the ROOT variable in order to match packages from an # alternative ROOT. best_version() { - local eroot - case "$EAPI" in - 0|1|2) - [[ " ${FEATURES} " == *" force-prefix "* ]] && \ - eroot=${ROOT%/}${EPREFIX}/ || eroot=${ROOT} - ;; - *) - eroot=${ROOT%/}${EPREFIX}/ - ;; - esac + local atom eroot host_root=false root=${ROOT} + if [[ $1 == --host-root ]] ; then + host_root=true + shift + fi + atom=$1 + shift + [ $# -gt 0 ] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if ${host_root} ; then + if ! ___eapi_best_version_and_has_version_support_--host-root; then + die "${FUNCNAME[0]}: option --host-root is not supported with EAPI ${EAPI}" + fi + root=/ + fi + + if ___eapi_has_prefix_variables; then + # [[ ${root} == / ]] would be ambiguous here, + # since both prefixes can share root=/ while + # having different EPREFIX offsets. + if ${host_root} ; then + eroot=${root%/}${PORTAGE_OVERRIDE_EPREFIX}/ + else + eroot=${root%/}${EPREFIX}/ + fi + else + eroot=${root} + fi if [[ -n $PORTAGE_IPC_DAEMON ]] ; then - "$PORTAGE_BIN_PATH"/ebuild-ipc best_version "${eroot}" "$1" + "$PORTAGE_BIN_PATH"/ebuild-ipc best_version "${eroot}" "${atom}" else - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" best_version "${eroot}" "$1" + "${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" best_version "${eroot}" "${atom}" fi local retval=$? case "${retval}" in 0|1) return ${retval} ;; + 2) + die "${FUNCNAME[0]}: invalid atom: ${atom}" + ;; *) - die "unexpected portageq exit code: ${retval}" + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi ;; esac } + +if ___eapi_has_master_repositories; then + master_repositories() { + local output repository=$1 retval + shift + [[ $# -gt 0 ]] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + "${PORTAGE_BIN_PATH}/ebuild-ipc" master_repositories "${EROOT}" "${repository}" + else + output=$("${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" master_repositories "${EROOT}" "${repository}") + fi + retval=$? + [[ -n ${output} ]] && echo "${output}" + case "${retval}" in + 0|1) + return ${retval} + ;; + 2) + die "${FUNCNAME[0]}: invalid repository: ${repository}" + ;; + *) + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi + ;; + esac + } +fi + +if ___eapi_has_repository_path; then + repository_path() { + local output repository=$1 retval + shift + [[ $# -gt 0 ]] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + "${PORTAGE_BIN_PATH}/ebuild-ipc" repository_path "${EROOT}" "${repository}" + else + output=$("${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" get_repo_path "${EROOT}" "${repository}") + fi + retval=$? + [[ -n ${output} ]] && echo "${output}" + case "${retval}" in + 0|1) + return ${retval} + ;; + 2) + die "${FUNCNAME[0]}: invalid repository: ${repository}" + ;; + *) + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi + ;; + esac + } +fi + +if ___eapi_has_available_eclasses; then + available_eclasses() { + local output repository=${PORTAGE_REPO_NAME} retval + [[ $# -gt 0 ]] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + "${PORTAGE_BIN_PATH}/ebuild-ipc" available_eclasses "${EROOT}" "${repository}" + else + output=$("${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" available_eclasses "${EROOT}" "${repository}") + fi + retval=$? + [[ -n ${output} ]] && echo "${output}" + case "${retval}" in + 0|1) + return ${retval} + ;; + 2) + die "${FUNCNAME[0]}: invalid repository: ${repository}" + ;; + *) + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi + ;; + esac + } +fi + +if ___eapi_has_eclass_path; then + eclass_path() { + local eclass=$1 output repository=${PORTAGE_REPO_NAME} retval + shift + [[ $# -gt 0 ]] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + "${PORTAGE_BIN_PATH}/ebuild-ipc" eclass_path "${EROOT}" "${repository}" "${eclass}" + else + output=$("${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" eclass_path "${EROOT}" "${repository}" "${eclass}") + fi + retval=$? + [[ -n ${output} ]] && echo "${output}" + case "${retval}" in + 0|1) + return ${retval} + ;; + 2) + die "${FUNCNAME[0]}: invalid repository: ${repository}" + ;; + *) + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi + ;; + esac + } +fi + +if ___eapi_has_license_path; then + license_path() { + local license=$1 output repository=${PORTAGE_REPO_NAME} retval + shift + [[ $# -gt 0 ]] && die "${FUNCNAME[0]}: unused argument(s): $*" + + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + "${PORTAGE_BIN_PATH}/ebuild-ipc" license_path "${EROOT}" "${repository}" "${license}" + else + output=$("${PORTAGE_BIN_PATH}/ebuild-helpers/portageq" license_path "${EROOT}" "${repository}" "${license}") + fi + retval=$? + [[ -n ${output} ]] && echo "${output}" + case "${retval}" in + 0|1) + return ${retval} + ;; + 2) + die "${FUNCNAME[0]}: invalid repository: ${repository}" + ;; + *) + if [[ -n ${PORTAGE_IPC_DAEMON} ]]; then + die "${FUNCNAME[0]}: unexpected ebuild-ipc exit code: ${retval}" + else + die "${FUNCNAME[0]}: unexpected portageq exit code: ${retval}" + fi + ;; + esac + } +fi + +if ___eapi_has_package_manager_build_user; then + package_manager_build_user() { + echo "${PORTAGE_BUILD_USER}" + } +fi + +if ___eapi_has_package_manager_build_group; then + package_manager_build_group() { + echo "${PORTAGE_BUILD_GROUP}" + } +fi diff --git a/portage_with_autodep/bin/portageq b/portage_with_autodep/bin/portageq index 280fe94..a50b805 100755 --- a/portage_with_autodep/bin/portageq +++ b/portage_with_autodep/bin/portageq @@ -1,15 +1,15 @@ #!/usr/bin/python -O -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals import signal import sys # This block ensures that ^C interrupts are handled quietly. try: - def exithandler(signum, frame): + def exithandler(signum, _frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) sys.exit(128 + signum) @@ -34,23 +34,22 @@ if os.environ.get("SANDBOX_ON") == "1": ":".join(filter(None, sandbox_write)) del sandbox_write -try: - import portage -except ImportError: - sys.path.insert(0, pym_path) - import portage -del pym_path - +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os from portage.eapi import eapi_has_repo_deps from portage.util import writemsg, writemsg_stdout -from portage.output import colormap +from portage.util._argparse import ArgumentParser portage.proxy.lazyimport.lazyimport(globals(), + 're', 'subprocess', '_emerge.Package:Package', '_emerge.RootConfig:RootConfig', + '_emerge.is_valid_package_atom:insert_category_into_atom', 'portage.dbapi._expand_new_virt:expand_new_virt', 'portage._sets.base:InternalPackageSet', + 'portage.xml.metadata:MetaDataXML' ) def eval_atom_use(atom): @@ -59,6 +58,10 @@ def eval_atom_use(atom): atom = atom.evaluate_conditionals(use) return atom +def uses_eroot(function): + function.uses_eroot = True + return function + #----------------------------------------------------------------------------- # # To add functionality to this tool, add a function below. @@ -80,13 +83,14 @@ def eval_atom_use(atom): # and will automaticly add a command by the same name as the function! # +@uses_eroot def has_version(argv): """<eroot> <category/package> Return code 0 if it's available, 1 otherwise. """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - return 2 + return 3 warnings = [] @@ -105,9 +109,7 @@ def has_version(argv): try: atom = portage.dep.Atom(argv[1], allow_repo=allow_repo, eapi=eapi) except portage.exception.InvalidAtom as e: - warnings.append( - portage._unicode_decode("QA Notice: %s: %s") % \ - ('has_version', e)) + warnings.append("QA Notice: %s: %s" % ('has_version', e)) atom = eval_atom_use(atom) if warnings: @@ -125,16 +127,16 @@ def has_version(argv): portage.writemsg("ERROR: Invalid atom: '%s'\n" % argv[1], noiselevel=-1) return 2 -has_version.uses_root = True +@uses_eroot def best_version(argv): """<eroot> <category/package> Returns category/package-version (without .ebuild). """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - return 2 + return 3 warnings = [] @@ -153,9 +155,7 @@ def best_version(argv): try: atom = portage.dep.Atom(argv[1], allow_repo=allow_repo, eapi=eapi) except portage.exception.InvalidAtom as e: - warnings.append( - portage._unicode_decode("QA Notice: %s: %s") % \ - ('best_version', e)) + warnings.append("QA Notice: %s: %s" % ('best_version', e)) atom = eval_atom_use(atom) if warnings: @@ -166,9 +166,9 @@ def best_version(argv): print(portage.best(mylist)) except KeyError: return 1 -best_version.uses_root = True +@uses_eroot def mass_best_version(argv): """<eroot> [<category/package>]+ Returns category/package-version (without .ebuild). @@ -178,23 +178,25 @@ def mass_best_version(argv): return 2 try: for pack in argv[1:]: - mylist=portage.db[argv[0]]["vartree"].dbapi.match(pack) - print(pack+":"+portage.best(mylist)) + mylist = portage.db[argv[0]]['vartree'].dbapi.match(pack) + print('%s:%s' % (pack, portage.best(mylist))) except KeyError: return 1 -mass_best_version.uses_root = True + +@uses_eroot def metadata(argv): if (len(argv) < 4): - print("ERROR: insufficient parameters!", file=sys.stderr) + print('ERROR: insufficient parameters!', file=sys.stderr) return 2 eroot, pkgtype, pkgspec = argv[0:3] metakeys = argv[3:] type_map = { - "ebuild":"porttree", - "binary":"bintree", - "installed":"vartree"} + 'ebuild': 'porttree', + 'binary': 'bintree', + 'installed': 'vartree' + } if pkgtype not in type_map: print("Unrecognized package type: '%s'" % pkgtype, file=sys.stderr) return 1 @@ -202,9 +204,9 @@ def metadata(argv): repo = portage.dep.dep_getrepo(pkgspec) pkgspec = portage.dep.remove_slot(pkgspec) try: - values = trees[eroot][type_map[pkgtype]].dbapi.aux_get( - pkgspec, metakeys, myrepo=repo) - writemsg_stdout(''.join('%s\n' % x for x in values), noiselevel=-1) + values = trees[eroot][type_map[pkgtype]].dbapi.aux_get( + pkgspec, metakeys, myrepo=repo) + writemsg_stdout(''.join('%s\n' % x for x in values), noiselevel=-1) except KeyError: print("Package not found: '%s'" % pkgspec, file=sys.stderr) return 1 @@ -216,8 +218,8 @@ Available keys: %s """ % ','.join(sorted(x for x in portage.auxdbkeys \ if not x.startswith('UNUSED_'))) -metadata.uses_root = True +@uses_eroot def contents(argv): """<eroot> <category/package> List the files that are installed for a given package, with @@ -238,8 +240,9 @@ def contents(argv): treetype="vartree", vartree=vartree) writemsg_stdout(''.join('%s\n' % x for x in sorted(db.getcontents())), noiselevel=-1) -contents.uses_root = True + +@uses_eroot def owners(argv): """<eroot> [<filename>]+ Given a list of files, print the packages that own the files and which @@ -253,7 +256,6 @@ def owners(argv): sys.stderr.flush() return 2 - from portage import catsplit, dblink eroot = argv[0] vardb = portage.db[eroot]["vartree"].dbapi root = portage.settings['ROOT'] @@ -319,8 +321,8 @@ def owners(argv): return 0 return 1 -owners.uses_root = True +@uses_eroot def is_protected(argv): """<eroot> <filename> Given a single filename, return code 0 if it's protected, 1 otherwise. @@ -366,8 +368,8 @@ def is_protected(argv): return 0 return 1 -is_protected.uses_root = True +@uses_eroot def filter_protected(argv): """<eroot> Read filenames from stdin and write them to stdout if they are protected. @@ -395,7 +397,6 @@ def filter_protected(argv): settings.get("CONFIG_PROTECT_MASK", "")) protect_obj = ConfigProtect(root, protect, protect_mask) - protected = 0 errors = 0 for line in sys.stdin: @@ -417,7 +418,6 @@ def filter_protected(argv): continue if protect_obj.isprotected(f): - protected += 1 out.write("%s\n" % filename) out.flush() @@ -426,8 +426,8 @@ def filter_protected(argv): return 0 -filter_protected.uses_root = True +@uses_eroot def best_visible(argv): """<eroot> [pkgtype] <atom> Returns category/package-version (without .ebuild). @@ -465,8 +465,7 @@ def best_visible(argv): noiselevel=-1) return 2 - root_config = RootConfig(portage.settings, - portage.db[eroot], None) + root_config = RootConfig(portage.settings, portage.db[eroot], None) if hasattr(db, "xmatch"): cpv_list = db.xmatch("match-all-cpv-only", atom) @@ -508,11 +507,11 @@ def best_visible(argv): writemsg_stdout("\n", noiselevel=-1) return 1 -best_visible.uses_root = True +@uses_eroot def mass_best_visible(argv): - """<root> [<type>] [<category/package>]+ + """<eroot> [<type>] [<category/package>]+ Returns category/package-version (without .ebuild). The pkgtype argument defaults to "ebuild" if unspecified, otherwise it must be one of ebuild, binary, or installed. @@ -535,9 +534,9 @@ def mass_best_visible(argv): best_visible([root, pkgtype, pack]) except KeyError: return 1 -mass_best_visible.uses_root = True +@uses_eroot def all_best_visible(argv): """<eroot> Returns all best_visible packages (without .ebuild). @@ -552,9 +551,9 @@ def all_best_visible(argv): mybest=portage.best(portage.db[argv[0]]["porttree"].dbapi.match(pkg)) if mybest: print(mybest) -all_best_visible.uses_root = True +@uses_eroot def match(argv): """<eroot> <atom> Returns a \\n separated list of category/package-version. @@ -583,17 +582,15 @@ def match(argv): require_metadata = atom.slot or atom.repo for cpv in vardb.cpv_all(): - if not portage.dep.extended_cp_match( - atom.cp, portage.cpv_getkey(cpv)): + if not portage.match_from_list(atom, [cpv]): continue if require_metadata: - slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) - - if atom.slot is not None and atom.slot != slot: + try: + cpv = vardb._pkg_str(cpv, atom.repo) + except (KeyError, portage.exception.InvalidData): continue - - if atom.repo is not None and atom.repo != repo: + if not portage.match_from_list(atom, [cpv]): continue results.append(cpv) @@ -603,8 +600,9 @@ def match(argv): results = vardb.match(atom) for cpv in results: print(cpv) -match.uses_root = True + +@uses_eroot def expand_virtual(argv): """<eroot> <atom> Returns a \\n separated list of atoms expanded from a @@ -639,9 +637,8 @@ def expand_virtual(argv): return os.EX_OK -expand_virtual.uses_root = True -def vdb_path(argv): +def vdb_path(_argv): """ Returns the path used for the var(installed) package database for the set environment/configuration options. @@ -651,56 +648,79 @@ def vdb_path(argv): out.flush() return os.EX_OK -def gentoo_mirrors(argv): +def gentoo_mirrors(_argv): """ Returns the mirrors set to use in the portage configuration. """ print(portage.settings["GENTOO_MIRRORS"]) -def portdir(argv): +@uses_eroot +def repositories_configuration(argv): + """<eroot> + Returns the configuration of repositories. + """ + if len(argv) < 1: + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 + sys.stdout.write(portage.db[argv[0]]["vartree"].settings.repositories.config_string()) + sys.stdout.flush() + +@uses_eroot +def repos_config(argv): + """ + <eroot> + This is an alias for the repositories_configuration command. + """ + return repositories_configuration(argv) + +def portdir(_argv): """ Returns the PORTDIR path. + Deprecated in favor of repositories_configuration command. """ + print("WARNING: 'portageq portdir' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) print(portage.settings["PORTDIR"]) -def config_protect(argv): +def config_protect(_argv): """ Returns the CONFIG_PROTECT paths. """ print(portage.settings["CONFIG_PROTECT"]) -def config_protect_mask(argv): +def config_protect_mask(_argv): """ Returns the CONFIG_PROTECT_MASK paths. """ print(portage.settings["CONFIG_PROTECT_MASK"]) -def portdir_overlay(argv): +def portdir_overlay(_argv): """ Returns the PORTDIR_OVERLAY path. + Deprecated in favor of repositories_configuration command. """ + print("WARNING: 'portageq portdir_overlay' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) print(portage.settings["PORTDIR_OVERLAY"]) -def pkgdir(argv): +def pkgdir(_argv): """ Returns the PKGDIR path. """ print(portage.settings["PKGDIR"]) -def distdir(argv): +def distdir(_argv): """ Returns the DISTDIR path. """ print(portage.settings["DISTDIR"]) -def colormap(argv): +def colormap(_argv): """ Display the color.map as environment variables. """ @@ -721,11 +741,15 @@ def envvar(argv): return 2 for arg in argv: + if arg in ("PORTDIR", "PORTDIR_OVERLAY", "SYNC"): + print("WARNING: 'portageq envvar %s' is deprecated. Use 'portageq repositories_configuration' instead." % arg, file=sys.stderr) if verbose: - print(arg +"='"+ portage.settings[arg] +"'") + print(arg + "=" + portage._shell_quote(portage.settings[arg])) else: print(portage.settings[arg]) + +@uses_eroot def get_repos(argv): """<eroot> Returns all repos with names (repo_name file) argv[0] = $EROOT @@ -733,25 +757,137 @@ def get_repos(argv): if len(argv) < 1: print("ERROR: insufficient parameters!") return 2 - print(" ".join(portage.db[argv[0]]["porttree"].dbapi.getRepositories())) + print(" ".join(reversed(portage.db[argv[0]]["vartree"].settings.repositories.prepos_order))) + + +@uses_eroot +def master_repositories(argv): + """<eroot> <repo_id>+ + Returns space-separated list of master repositories for specified repository. + """ + if len(argv) < 2: + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 + for arg in argv[1:]: + if portage.dep._repo_name_re.match(arg) is None: + print("ERROR: invalid repository: %s" % arg, file=sys.stderr) + return 2 + try: + repo = portage.db[argv[0]]["vartree"].settings.repositories[arg] + except KeyError: + print("") + return 1 + else: + print(" ".join(x.name for x in repo.masters)) -get_repos.uses_root = True +@uses_eroot +def master_repos(argv): + """<eroot> <repo_id>+ + This is an alias for the master_repositories command. + """ + return master_repositories(argv) +@uses_eroot def get_repo_path(argv): """<eroot> <repo_id>+ Returns the path to the repo named argv[1], argv[0] = $EROOT """ if len(argv) < 2: - print("ERROR: insufficient parameters!") - return 2 + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 for arg in argv[1:]: - path = portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(arg) + if portage.dep._repo_name_re.match(arg) is None: + print("ERROR: invalid repository: %s" % arg, file=sys.stderr) + return 2 + path = portage.db[argv[0]]["vartree"].settings.repositories.treemap.get(arg) if path is None: - path = "" + print("") + return 1 print(path) -get_repo_path.uses_root = True +@uses_eroot +def available_eclasses(argv): + """<eroot> <repo_id>+ + Returns space-separated list of available eclasses for specified repository. + """ + if len(argv) < 2: + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 + for arg in argv[1:]: + if portage.dep._repo_name_re.match(arg) is None: + print("ERROR: invalid repository: %s" % arg, file=sys.stderr) + return 2 + try: + repo = portage.db[argv[0]]["vartree"].settings.repositories[arg] + except KeyError: + print("") + return 1 + else: + print(" ".join(sorted(repo.eclass_db.eclasses))) + + +@uses_eroot +def eclass_path(argv): + """<eroot> <repo_id> <eclass>+ + Returns the path to specified eclass for specified repository. + """ + if len(argv) < 3: + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 + if portage.dep._repo_name_re.match(argv[1]) is None: + print("ERROR: invalid repository: %s" % argv[1], file=sys.stderr) + return 2 + try: + repo = portage.db[argv[0]]["vartree"].settings.repositories[argv[1]] + except KeyError: + print("") + return 1 + else: + retval = 0 + for arg in argv[2:]: + try: + eclass = repo.eclass_db.eclasses[arg] + except KeyError: + print("") + retval = 1 + else: + print(eclass.location) + return retval + + +@uses_eroot +def license_path(argv): + """<eroot> <repo_id> <license>+ + Returns the path to specified license for specified repository. + """ + if len(argv) < 3: + print("ERROR: insufficient parameters!", file=sys.stderr) + return 3 + if portage.dep._repo_name_re.match(argv[1]) is None: + print("ERROR: invalid repository: %s" % argv[1], file=sys.stderr) + return 2 + try: + repo = portage.db[argv[0]]["vartree"].settings.repositories[argv[1]] + except KeyError: + print("") + return 1 + else: + retval = 0 + for arg in argv[2:]: + eclass_path = "" + paths = reversed([os.path.join(x.location, 'licenses', arg) for x in list(repo.masters) + [repo]]) + for path in paths: + if os.path.exists(path): + eclass_path = path + break + if eclass_path == "": + retval = 1 + print(eclass_path) + return retval + + +@uses_eroot def list_preserved_libs(argv): """<eroot> Print a list of libraries preserved during a package update in the form @@ -773,21 +909,296 @@ def list_preserved_libs(argv): msg.append('\n') writemsg_stdout(''.join(msg), noiselevel=-1) return rValue -list_preserved_libs.uses_root = True + + +class MaintainerEmailMatcher(object): + def __init__(self, maintainer_emails): + self._re = re.compile("^(%s)$" % "|".join(maintainer_emails)) + + def __call__(self, metadata_xml): + match = False + matcher = self._re.match + for x in metadata_xml.maintainers(): + if x.email is not None and matcher(x.email) is not None: + match = True + break + return match + +class HerdMatcher(object): + def __init__(self, herds): + self._herds = frozenset(herds) + + def __call__(self, metadata_xml): + herds = self._herds + return any(x in herds for x in metadata_xml.herds()) + + +def pquery(parser, opts, args): + """[options] [atom]+ + Emulates a subset of Pkgcore's pquery tool. + """ + + portdb = portage.db[portage.root]['porttree'].dbapi + root_config = RootConfig(portdb.settings, + portage.db[portage.root], None) + + def _pkg(cpv, repo_name): + try: + metadata = dict(zip( + Package.metadata_keys, + portdb.aux_get(cpv, + Package.metadata_keys, + myrepo=repo_name))) + except KeyError: + raise portage.exception.PackageNotFound(cpv) + return Package(built=False, cpv=cpv, + installed=False, metadata=metadata, + root_config=root_config, + type_name="ebuild") + + need_metadata = False + atoms = [] + for arg in args: + if "/" not in arg.split(":")[0]: + atom = insert_category_into_atom(arg, '*') + if atom is None: + writemsg("ERROR: Invalid atom: '%s'\n" % arg, + noiselevel=-1) + return 2 + else: + atom = arg + + try: + atom = portage.dep.Atom(atom, allow_wildcard=True, allow_repo=True) + except portage.exception.InvalidAtom: + writemsg("ERROR: Invalid atom: '%s'\n" % arg, + noiselevel=-1) + return 2 + + if atom.slot is not None: + need_metadata = True + + atoms.append(atom) + + if "*/*" in atoms: + del atoms[:] + need_metadata = False + + if not opts.no_filters: + need_metadata = True + + xml_matchers = [] + if opts.maintainer_email: + maintainer_emails = [] + for x in opts.maintainer_email: + maintainer_emails.extend(x.split(",")) + xml_matchers.append(MaintainerEmailMatcher(maintainer_emails)) + if opts.herd is not None: + herds = [] + for x in opts.herd: + herds.extend(x.split(",")) + xml_matchers.append(HerdMatcher(herds)) + + repos = [] + if opts.all_repos: + repos.extend(portdb.repositories.get_repo_for_location(location) + for location in portdb.porttrees) + elif opts.repo is not None: + repos.append(portdb.repositories[opts.repo]) + else: + repos.append(portdb.repositories.mainRepo()) + + if not atoms: + names = None + categories = list(portdb.categories) + else: + category_wildcard = False + name_wildcard = False + categories = [] + names = [] + for atom in atoms: + category, name = portage.catsplit(atom.cp) + categories.append(category) + names.append(name) + if "*" in category: + category_wildcard = True + if "*" in name: + name_wildcard = True + + if category_wildcard: + categories = list(portdb.categories) + else: + categories = list(set(categories)) + + if name_wildcard: + names = None + else: + names = sorted(set(names)) + + no_version = opts.no_version + categories.sort() + + for category in categories: + if names is None: + cp_list = portdb.cp_all(categories=(category,)) + else: + cp_list = [category + "/" + name for name in names] + for cp in cp_list: + matches = [] + for repo in repos: + match = True + if xml_matchers: + metadata_xml_path = os.path.join( + repo.location, cp, 'metadata.xml') + try: + metadata_xml = MetaDataXML(metadata_xml_path, None) + except (EnvironmentError, SyntaxError): + match = False + else: + for matcher in xml_matchers: + if not matcher(metadata_xml): + match = False + break + if not match: + continue + cpv_list = portdb.cp_list(cp, mytree=[repo.location]) + if atoms: + for cpv in cpv_list: + pkg = None + for atom in atoms: + if atom.repo is not None and \ + atom.repo != repo.name: + continue + if not portage.match_from_list(atom, [cpv]): + continue + if need_metadata: + if pkg is None: + try: + pkg = _pkg(cpv, repo.name) + except portage.exception.PackageNotFound: + continue + + if not (opts.no_filters or pkg.visible): + continue + if not portage.match_from_list(atom, [pkg]): + continue + matches.append(cpv) + break + if no_version and matches: + break + elif opts.no_filters: + matches.extend(cpv_list) + else: + for cpv in cpv_list: + try: + pkg = _pkg(cpv, repo.name) + except portage.exception.PackageNotFound: + continue + else: + if pkg.visible: + matches.append(cpv) + if no_version: + break + + if no_version and matches: + break + + if not matches: + continue + + if no_version: + writemsg_stdout("%s\n" % (cp,), noiselevel=-1) + else: + matches = list(set(matches)) + portdb._cpv_sort_ascending(matches) + for cpv in matches: + writemsg_stdout("%s\n" % (cpv,), noiselevel=-1) + + return os.EX_OK + #----------------------------------------------------------------------------- # # DO NOT CHANGE CODE BEYOND THIS POINT - IT'S NOT NEEDED! # -if not portage.const._ENABLE_PRESERVE_LIBS: - del list_preserved_libs - -non_commands = frozenset(['elog', 'eval_atom_use', - 'exithandler', 'expand_new_virt', 'main', - 'usage', 'writemsg', 'writemsg_stdout']) +non_commands = frozenset(['elog', 'eval_atom_use', 'exithandler', 'main', 'usage', 'uses_eroot']) commands = sorted(k for k, v in globals().items() \ - if k not in non_commands and isinstance(v, types.FunctionType)) + if k not in non_commands and isinstance(v, types.FunctionType) and v.__module__ == "__main__") + + +def add_pquery_arguments(parser): + pquery_option_groups = ( + ( + 'Repository matching options', + ( + { + "longopt": "--no-filters", + "action": "store_true", + "help": "no visibility filters (ACCEPT_KEYWORDS, package masking, etc)" + }, + { + "longopt": "--repo", + "help": "repo to use (default is PORTDIR if omitted)" + }, + { + "longopt": "--all-repos", + "help": "search all repos" + } + ) + ), + ( + 'Package matching options', + ( + { + "longopt": "--herd", + "action": "append", + "help": "exact match on a herd" + }, + { + "longopt": "--maintainer-email", + "action": "append", + "help": "comma-separated list of maintainer email regexes to search for" + } + ) + ), + ( + 'Output formatting', + ( + { + "shortopt": "-n", + "longopt": "--no-version", + "action": "store_true", + "help": "collapse multiple matching versions together" + }, + ) + ), + ) + + for group_title, opt_data in pquery_option_groups: + arg_group = parser.add_argument_group(group_title) + for opt_info in opt_data: + pargs = [] + try: + pargs.append(opt_info["shortopt"]) + except KeyError: + pass + try: + pargs.append(opt_info["longopt"]) + except KeyError: + pass + + kwargs = {} + try: + kwargs["action"] = opt_info["action"] + except KeyError: + pass + try: + kwargs["help"] = opt_info["help"] + except KeyError: + pass + arg_group.add_argument(*pargs, **portage._native_kwargs(kwargs)) + def usage(argv): print(">>> Portage information query tool") @@ -800,7 +1211,7 @@ def usage(argv): # Show our commands -- we do this by scanning the functions in this # file, and formatting each functions documentation. # - help_mode = '--help' in sys.argv + help_mode = '--help' in argv for name in commands: # Drop non-functions obj = globals()[name] @@ -814,12 +1225,21 @@ def usage(argv): lines = doc.lstrip("\n").split("\n") print(" " + name + " " + lines[0].strip()) - if (len(sys.argv) > 1): + if len(argv) > 1: if (not help_mode): lines = lines[:-1] for line in lines[1:]: print(" " + line.strip()) - if (len(sys.argv) == 1): + + print() + print('Pkgcore pquery compatible options:') + print() + parser = ArgumentParser(add_help=False, + usage='portageq pquery [options] [atom ...]') + add_pquery_arguments(parser) + parser.print_help() + + if len(argv) == 1: print("\nRun portageq with --help for info") atom_validate_strict = "EBUILD_PHASE" in os.environ @@ -838,52 +1258,84 @@ else: def elog(elog_funcname, lines): pass -def main(): +def main(argv): + + argv = portage._decode_argv(argv) nocolor = os.environ.get('NOCOLOR') if nocolor in ('yes', 'true'): portage.output.nocolor() - if len(sys.argv) < 2: - usage(sys.argv) - sys.exit(os.EX_USAGE) + parser = ArgumentParser(add_help=False) - for x in sys.argv: - if x in ("-h", "--help"): - usage(sys.argv) - sys.exit(os.EX_OK) - elif x == "--version": - print("Portage", portage.VERSION) - sys.exit(os.EX_OK) - - cmd = sys.argv[1] - function = globals().get(cmd) - if function is None or cmd not in commands: - usage(sys.argv) + # used by envvar + parser.add_argument("-v", dest="verbose", action="store_true") + + actions = parser.add_argument_group('Actions') + actions.add_argument("-h", "--help", action="store_true") + actions.add_argument("--version", action="store_true") + + add_pquery_arguments(parser) + + opts, args = parser.parse_known_args(argv[1:]) + + if opts.help: + usage(argv) + return os.EX_OK + elif opts.version: + print("Portage", portage.VERSION) + return os.EX_OK + + cmd = None + if args and args[0] in commands: + cmd = args[0] + + if cmd == 'pquery': + cmd = None + args = args[1:] + + if cmd is None: + return pquery(parser, opts, args) + + if opts.verbose: + # used by envvar + args.append("-v") + + argv = argv[:1] + args + + if len(argv) < 2: + usage(argv) sys.exit(os.EX_USAGE) + function = globals()[cmd] - uses_root = getattr(function, "uses_root", False) and len(sys.argv) > 2 - if uses_root: - if not os.path.isdir(sys.argv[2]): - sys.stderr.write("Not a directory: '%s'\n" % sys.argv[2]) + uses_eroot = getattr(function, "uses_eroot", False) and len(argv) > 2 + if uses_eroot: + if not os.path.isdir(argv[2]): + sys.stderr.write("Not a directory: '%s'\n" % argv[2]) sys.stderr.write("Run portageq with --help for info\n") sys.stderr.flush() sys.exit(os.EX_USAGE) - eprefix = portage.const.EPREFIX - eroot = portage.util.normalize_path(sys.argv[2]) + eprefix = portage.settings["EPREFIX"] + eroot = portage.util.normalize_path(argv[2]) + if eprefix: - root = eroot[:1-len(eprefix)] + if not eroot.endswith(eprefix): + sys.stderr.write("ERROR: This version of portageq" + " only supports <eroot>s ending in" + " '%s'. The provided <eroot>, '%s'," + " doesn't.\n" % (eprefix, eroot)) + sys.stderr.flush() + sys.exit(os.EX_USAGE) + root = eroot[:1 - len(eprefix)] else: root = eroot + os.environ["ROOT"] = root - args = sys.argv[2:] - if args and isinstance(args[0], bytes): - for i in range(len(args)): - args[i] = portage._unicode_decode(args[i]) + args = argv[2:] try: - if uses_root: + if uses_eroot: args[0] = portage.settings['EROOT'] retval = function(args) if retval: @@ -904,6 +1356,7 @@ def main(): portage.writemsg("\nPlease use a more specific atom.\n", noiselevel=-1) sys.exit(1) -main() +if __name__ == '__main__': + sys.exit(main(sys.argv)) #----------------------------------------------------------------------------- diff --git a/portage_with_autodep/bin/quickpkg b/portage_with_autodep/bin/quickpkg index d908c03..cf5800c 100755 --- a/portage_with_autodep/bin/quickpkg +++ b/portage_with_autodep/bin/quickpkg @@ -1,33 +1,31 @@ #!/usr/bin/python -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import errno import math -import optparse import signal import sys import tarfile -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os from portage import xpak from portage.dbapi.dep_expand import dep_expand -from portage.dep import Atom, extended_cp_match, use_reduce +from portage.dep import Atom, use_reduce from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData, InvalidDependString, PackageSetNotFound, PermissionDenied) from portage.util import ConfigProtect, ensure_dirs, shlex_split from portage.dbapi.vartree import dblink, tar_contents from portage.checksum import perform_md5 from portage._sets import load_default_config, SETPREFIX +from portage.util._argparse import ArgumentParser def quickpkg_atom(options, infos, arg, eout): settings = portage.settings @@ -68,11 +66,14 @@ def quickpkg_atom(options, infos, arg, eout): bintree.prevent_collision(cpv) dblnk = vardb._dblink(cpv) have_lock = False - try: - dblnk.lockdb() - have_lock = True - except PermissionDenied: - pass + + if "__PORTAGE_INHERIT_VARDB_LOCK" not in settings: + try: + dblnk.lockdb() + have_lock = True + except PermissionDenied: + pass + try: if not dblnk.exists(): # unmerged by a concurrent process @@ -201,16 +202,15 @@ def quickpkg_extended_atom(options, infos, atom, eout): atoms.append(cpv_atom) continue - if not extended_cp_match(atom.cp, cpv_atom.cp): + if not portage.match_from_list(atom, [cpv]): continue if require_metadata: - slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) - - if atom.slot and atom.slot != slot: + try: + cpv = vardb._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): continue - - if atom.repo and atom.repo != repo: + if not portage.match_from_list(atom, [cpv]): continue atoms.append(cpv_atom) @@ -289,29 +289,28 @@ def quickpkg_main(options, args, eout): if __name__ == "__main__": usage = "quickpkg [options] <list of package atoms or package sets>" - parser = optparse.OptionParser(usage=usage) - parser.add_option("--umask", + parser = ArgumentParser(usage=usage) + parser.add_argument("--umask", default="0077", help="umask used during package creation (default is 0077)") - parser.add_option("--ignore-default-opts", + parser.add_argument("--ignore-default-opts", action="store_true", help="do not use the QUICKPKG_DEFAULT_OPTS environment variable") - parser.add_option("--include-config", - type="choice", + parser.add_argument("--include-config", choices=["y","n"], default="n", metavar="<y|n>", help="include all files protected by CONFIG_PROTECT (as a security precaution, default is 'n')") - parser.add_option("--include-unmodified-config", - type="choice", + parser.add_argument("--include-unmodified-config", choices=["y","n"], default="n", metavar="<y|n>", help="include files protected by CONFIG_PROTECT that have not been modified since installation (as a security precaution, default is 'n')") - options, args = parser.parse_args(sys.argv[1:]) + options, args = parser.parse_known_args(sys.argv[1:]) if not options.ignore_default_opts: - default_opts = portage.settings.get("QUICKPKG_DEFAULT_OPTS","").split() - options, args = parser.parse_args(default_opts + sys.argv[1:]) + default_opts = shlex_split( + portage.settings.get("QUICKPKG_DEFAULT_OPTS", "")) + options, args = parser.parse_known_args(default_opts + sys.argv[1:]) if not args: parser.error("no packages atoms given") try: diff --git a/portage_with_autodep/bin/regenworld b/portage_with_autodep/bin/regenworld index 3199fdf..f74b3dd 100755 --- a/portage_with_autodep/bin/regenworld +++ b/portage_with_autodep/bin/regenworld @@ -1,17 +1,15 @@ #!/usr/bin/python -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import sys -try: - import portage -except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage - +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True from portage import os from portage._sets.files import StaticFileSet, WorldSelectedSet diff --git a/portage_with_autodep/bin/repoman b/portage_with_autodep/bin/repoman index fd87847..d1542e9 100755 --- a/portage_with_autodep/bin/repoman +++ b/portage_with_autodep/bin/repoman @@ -1,20 +1,19 @@ #!/usr/bin/python -O -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Next to do: dep syntax checking in mask files # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems) # that last one is tricky because multiple profiles need to be checked. -from __future__ import print_function +from __future__ import print_function, unicode_literals -import calendar +import codecs import copy import errno import formatter import io import logging -import optparse import re import signal import stat @@ -24,28 +23,29 @@ import tempfile import textwrap import time import platform - -try: - from urllib.request import urlopen as urllib_request_urlopen -except ImportError: - from urllib import urlopen as urllib_request_urlopen - from itertools import chain from stat import S_ISDIR try: - import portage + from urllib.parse import urlparse except ImportError: - from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage + from urlparse import urlparse + +from os import path as osp +pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") +sys.path.insert(0, pym_path) +import portage +portage._internal_caller = True portage._disable_legacy_globals() -portage.dep._internal_warnings = True try: import xml.etree.ElementTree from xml.parsers.expat import ExpatError -except (ImportError, SystemError): +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] from portage.output import EOutput out = EOutput() @@ -54,9 +54,9 @@ except (ImportError, SystemError): sys.exit(1) from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings from portage import _unicode_encode +import repoman.checks from repoman.checks import run_checks from repoman import utilities from repoman.herdbase import make_herd_base @@ -65,18 +65,18 @@ from _emerge.RootConfig import RootConfig from _emerge.userquery import userquery import portage.checksum import portage.const +import portage.repository.config from portage import cvstree, normalize_path from portage import util -from portage.exception import (FileNotFound, MissingParameter, +from portage.exception import (FileNotFound, InvalidAtom, MissingParameter, ParseError, PermissionDenied) -from portage.manifest import _prohibited_filename_chars_re as \ - disallowed_filename_chars_re +from portage.dep import Atom from portage.process import find_binary, spawn from portage.output import bold, create_color_func, \ green, nocolor, red from portage.output import ConsoleStyleFile, StyleWriter from portage.util import writemsg_level -from portage.util._desktop_entry import validate_desktop_entry +from portage.util._argparse import ArgumentParser from portage.package.ebuild.digestgen import digestgen from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use @@ -89,6 +89,7 @@ util.initialize_logger() max_desc_len = 100 allowed_filename_chars="a-zA-Z0-9._-+:" pv_toolong_re = re.compile(r'[0-9]{19,}') +GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?' bad = create_color_func("BAD") # A sane umask is needed for files that portage creates. @@ -112,41 +113,14 @@ def err(txt): warn(txt) sys.exit(1) -def exithandler(signum=None, frame=None): +def exithandler(signum=None, _frame=None): logging.fatal("Interrupted; exiting...") if signum is None: sys.exit(1) else: sys.exit(128 + signum) -signal.signal(signal.SIGINT,exithandler) - -class RepomanHelpFormatter(optparse.IndentedHelpFormatter): - """Repoman needs it's own HelpFormatter for now, because the default ones - murder the help text.""" - - def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1): - optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first) - - def format_description(self, description): - return description - -class RepomanOptionParser(optparse.OptionParser): - """Add the on_tail function, ruby has it, optionParser should too - """ - - def __init__(self, *args, **kwargs): - optparse.OptionParser.__init__(self, *args, **kwargs) - self.tail = "" - - def on_tail(self, description): - self.tail += description - - def format_help(self, formatter=None): - result = optparse.OptionParser.format_help(self, formatter) - result += self.tail - return result - +signal.signal(signal.SIGINT, exithandler) def ParseArgs(argv, qahelp): """This function uses a customized optionParser to parse command line arguments for repoman @@ -157,8 +131,7 @@ def ParseArgs(argv, qahelp): (opts, args), just like a call to parser.parse_args() """ - if argv and isinstance(argv[0], bytes): - argv = [portage._unicode_decode(x) for x in argv] + argv = portage._decode_argv(argv) modes = { 'commit' : 'Run a scan then commit changes', @@ -168,102 +141,103 @@ def ParseArgs(argv, qahelp): 'help' : 'Show this screen', 'manifest' : 'Generate a Manifest (fetches files if necessary)', 'manifest-check' : 'Check Manifests for missing or incorrect digests', - 'scan' : 'Scan directory tree for QA issues' + 'scan' : 'Scan directory tree for QA issues' } mode_keys = list(modes) mode_keys.sort() - parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]") - parser.description = green(" ".join((os.path.basename(argv[0]), "1.2"))) - parser.description += "\nCopyright 1999-2007 Gentoo Foundation" - parser.description += "\nDistributed under the terms of the GNU General Public License v2" - parser.description += "\nmodes: " + " | ".join(map(green,mode_keys)) + parser = ArgumentParser(usage="repoman [options] [mode]", + description="Modes: %s" % " | ".join(mode_keys), + epilog="For more help consult the man page.") - parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False, + parser.add_argument('-a', '--ask', dest='ask', action='store_true', default=False, help='Request a confirmation before commiting') - parser.add_option('-m', '--commitmsg', dest='commitmsg', + parser.add_argument('-m', '--commitmsg', dest='commitmsg', help='specify a commit message on the command line') - parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile', + parser.add_argument('-M', '--commitmsgfile', dest='commitmsgfile', help='specify a path to a file that contains a commit message') - parser.add_option('--digest', - type='choice', choices=('y', 'n'), metavar='<y|n>', + parser.add_argument('--digest', + choices=('y', 'n'), metavar='<y|n>', help='Automatically update Manifest digests for modified files') - parser.add_option('-p', '--pretend', dest='pretend', default=False, + parser.add_argument('-p', '--pretend', dest='pretend', default=False, action='store_true', help='don\'t commit or fix anything; just show what would be done') - - parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0, + + parser.add_argument('-q', '--quiet', dest="quiet", action="count", default=0, help='do not print unnecessary messages') - parser.add_option( - '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>", + parser.add_argument( + '--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>", help='for commit mode, call echangelog if ChangeLog is unmodified (or ' 'regardless of modification if \'force\' is specified)') - parser.add_option('-f', '--force', dest='force', default=False, action='store_true', + parser.add_argument('--experimental-inherit', choices=('y', 'n'), + metavar="<y|n>", default='n', + help='Enable experimental inherit.missing checks which may misbehave' + ' when the internal eclass database becomes outdated') + + parser.add_argument('-f', '--force', dest='force', default=False, action='store_true', help='Commit with QA violations') - parser.add_option('--vcs', dest='vcs', + parser.add_argument('--vcs', dest='vcs', help='Force using specific VCS instead of autodetection') - parser.add_option('-v', '--verbose', dest="verbosity", action='count', + parser.add_argument('-v', '--verbose', dest="verbosity", action='count', help='be very verbose in output', default=0) - parser.add_option('-V', '--version', dest='version', action='store_true', + parser.add_argument('-V', '--version', dest='version', action='store_true', help='show version info') - parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true', + parser.add_argument('-x', '--xmlparse', dest='xml_parse', action='store_true', default=False, help='forces the metadata.xml parse check to be carried out') - parser.add_option( - '--if-modified', type='choice', choices=('y', 'n'), default='n', + parser.add_argument( + '--if-modified', choices=('y', 'n'), default='n', metavar="<y|n>", help='only check packages that have uncommitted modifications') - parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true', + parser.add_argument('-i', '--ignore-arches', dest='ignore_arches', action='store_true', default=False, help='ignore arch-specific failures (where arch != host)') - parser.add_option("--ignore-default-opts", + parser.add_argument("--ignore-default-opts", action="store_true", help="do not use the REPOMAN_DEFAULT_OPTS environment variable") - parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true', + parser.add_argument('-I', '--ignore-masked', dest='ignore_masked', action='store_true', default=False, help='ignore masked packages (not allowed with commit mode)') - parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true', + parser.add_argument('--include-arches', dest='include_arches', + metavar='ARCHES', action='append', + help='A space separated list of arches used to ' + 'filter the selection of profiles for dependency checks') + + parser.add_argument('-d', '--include-dev', dest='include_dev', action='store_true', default=False, help='include dev profiles in dependency checks') - parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true', + parser.add_argument('-e', '--include-exp-profiles', choices=('y', 'n'), + default=False, help='include exp profiles in dependency checks', + metavar='<y|n>') + + parser.add_argument('--unmatched-removal', dest='unmatched_removal', action='store_true', default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms') - parser.add_option('--without-mask', dest='without_mask', action='store_true', + parser.add_argument('--without-mask', dest='without_mask', action='store_true', default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)') - parser.add_option('--mode', type='choice', dest='mode', choices=list(modes), + parser.add_argument('--mode', dest='mode', choices=mode_keys, help='specify which mode repoman will run in (default=full)') - parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n")) - - for k in mode_keys: - parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k])) - - parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n")) - - sorted_qa = list(qahelp) - sorted_qa.sort() - for k in sorted_qa: - parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k])) - - opts, args = parser.parse_args(argv[1:]) + opts, args = parser.parse_known_args(argv[1:]) if not opts.ignore_default_opts: - default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split() + default_opts = portage.util.shlex_split( + repoman_settings.get("REPOMAN_DEFAULT_OPTS", "")) if default_opts: - opts, args = parser.parse_args(default_opts + sys.argv[1:]) + opts, args = parser.parse_known_args(default_opts + sys.argv[1:]) if opts.mode == 'help': parser.print_help(short=False) @@ -278,16 +252,10 @@ def ParseArgs(argv, qahelp): if not opts.mode: opts.mode = 'full' - + if opts.mode == 'ci': opts.mode = 'commit' # backwards compat shortcut - if opts.mode == 'commit' and not (opts.force or opts.pretend): - if opts.ignore_masked: - parser.error('Commit mode and --ignore-masked are not compatible') - if opts.without_mask: - parser.error('Commit mode and --without-mask are not compatible') - # Use the verbosity and quiet options to fiddle with the loglevel appropriately for val in range(opts.verbosity): logger = logging.getLogger() @@ -297,100 +265,99 @@ def ParseArgs(argv, qahelp): logger = logging.getLogger() logger.setLevel(logger.getEffectiveLevel() + 10) + if opts.mode == 'commit' and not (opts.force or opts.pretend): + if opts.ignore_masked: + opts.ignore_masked = False + logging.warn('Commit mode automatically disables --ignore-masked') + if opts.without_mask: + opts.without_mask = False + logging.warn('Commit mode automatically disables --without-mask') + return (opts, args) -qahelp={ - "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file", - "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file", - "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)", - "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory", - "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified", - "changelog.missing":"Missing ChangeLog files", - "ebuild.notadded":"Ebuilds that exist but have not been added to cvs", - "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety", - "changelog.notadded":"ChangeLogs that exist but have not been added to cvs", - "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)", - "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit", - "file.size":"Files in the files directory must be under 20 KiB", - "file.size.fatal":"Files in the files directory must be under 60 KiB", - "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars, - "file.UTF8":"File is not UTF8 compliant", - "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf", - "inherit.deprecated":"Ebuild inherits a deprecated eclass", - "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass", - "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass", - "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch", - "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable", - "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS", - "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask", - "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable", - "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable", - "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable", - "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len, - "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", - "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI", - "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI", - "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)", - "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value", - "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable", - "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable", - "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)", - "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)", - "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)", - "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)", - "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)", - "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)", - "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch", - "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch", - "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch", - "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch", - "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch", - "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch", - "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.", - "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)", - "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)", - "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)", - "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)", - "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)", - "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)", - "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)", - "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)", - "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)", - "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)", - "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)", - "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)", - "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI", - "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure", - "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.", - "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.", - "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set", - "variable.readonly":"Assigning a readonly variable", - "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers", - "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.", - "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.", - "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file", - "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE", - "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)", - "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.", - "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found", - "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)", - "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.", - "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.", - "digest.assumed":"Existing digest must be assumed correct (Package level only)", - "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest", - "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI", - "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully", - "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style", - "ebuild.badheader":"This ebuild has a malformed header", - "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass", - "manifest.bad":"Manifest has missing or incorrect digests", - "metadata.missing":"Missing metadata.xml files", - "metadata.bad":"Bad metadata.xml files", - "metadata.warning":"Warnings in metadata.xml files", - "portage.internal":"The ebuild uses an internal Portage function", - "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)", - "usage.obsolete":"The ebuild makes use of an obsolete construct", - "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org" +qahelp = { + "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file", + "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)", + "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory", + "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified", + "changelog.missing": "Missing ChangeLog files", + "ebuild.notadded": "Ebuilds that exist but have not been added to cvs", + "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety", + "changelog.notadded": "ChangeLogs that exist but have not been added to cvs", + "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)", + "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)", + "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch", + "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch", + "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)", + "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)", + "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)", + "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit", + "file.size": "Files in the files directory must be under 20 KiB", + "file.size.fatal": "Files in the files directory must be under 60 KiB", + "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars, + "file.UTF8": "File is not UTF8 compliant", + "inherit.deprecated": "Ebuild inherits a deprecated eclass", + "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it", + "inherit.unused": "Ebuild inherits an eclass but does not use it", + "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass", + "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass", + "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch", + "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable", + "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS", + "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask", + "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable", + "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable", + "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable", + "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len, + "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", + "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI", + "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI", + "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)", + "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value", + "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable", + "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable", + "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.", + "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)", + "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)", + "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)", + "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)", + "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)", + "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)", + "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI", + "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure", + "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.", + "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.", + "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set", + "variable.readonly": "Assigning a readonly variable", + "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers", + "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.", + "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.", + "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file", + "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE", + "IUSE.rubydeprecated": "The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore", + "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.", + "LICENSE.deprecated": "This ebuild is listing a deprecated license.", + "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found", + "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)", + "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.", + "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.", + "digest.assumed": "Existing digest must be assumed correct (Package level only)", + "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest", + "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI", + "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully", + "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style", + "ebuild.badheader": "This ebuild has a malformed header", + "manifest.bad": "Manifest has missing or incorrect digests", + "metadata.missing": "Missing metadata.xml files", + "metadata.bad": "Bad metadata.xml files", + "metadata.warning": "Warnings in metadata.xml files", + "portage.internal": "The ebuild uses an internal Portage function or variable", + "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings", + "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings", + "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)", + "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.", + "usage.obsolete": "The ebuild makes use of an obsolete construct", + "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org" } qacats = list(qahelp) @@ -404,37 +371,39 @@ qawarnings = set(( "digest.unused", "ebuild.notadded", "ebuild.nesteddie", -"desktop.invalid", -"DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked", -"DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev", -"DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev", -"DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde", +"dependency.badmasked", +"dependency.badindev", +"dependency.badmaskedindev", +"dependency.badtilde", "DESCRIPTION.toolong", "EAPI.deprecated", "HOMEPAGE.virtual", +"LICENSE.deprecated", "LICENSE.virtual", "KEYWORDS.dropped", "KEYWORDS.stupid", "KEYWORDS.missing", -"IUSE.undefined", "PDEPEND.suspect", "RDEPEND.implicit", "RDEPEND.suspect", +"virtual.suspect", "RESTRICT.invalid", "ebuild.minorsyn", "ebuild.badheader", "ebuild.patches", "file.size", -"inherit.autotools", +"inherit.unused", "inherit.deprecated", "java.eclassesnotused", "wxwidgets.eclassnotused", "metadata.warning", "portage.internal", +"repo.eapi.deprecated", "usage.obsolete", "upstream.workaround", "LIVEVCS.stable", "LIVEVCS.unmasked", +"IUSE.rubydeprecated", )) non_ascii_re = re.compile(r'[^\x00-\x7f]') @@ -443,7 +412,7 @@ missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"] allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_")) allvars.update(Package.metadata_keys) allvars = sorted(allvars) -commitmessage=None +commitmessage = None for x in missingvars: x += ".missing" if x not in qacats: @@ -452,19 +421,10 @@ for x in missingvars: qawarnings.add(x) valid_restrict = frozenset(["binchecks", "bindist", - "fetch", "installsources", "mirror", - "primaryuri", "strip", "test", "userpriv"]) - -live_eclasses = frozenset([ - "bzr", - "cvs", - "darcs", - "git", - "git-2", - "mercurial", - "subversion", - "tla", -]) + "fetch", "installsources", "mirror", "preserve-libs", + "primaryuri", "splitdebug", "strip", "test", "userpriv"]) + +live_eclasses = portage.const.LIVE_ECLASSES suspect_rdepend = frozenset([ "app-arch/cabextract", @@ -482,7 +442,10 @@ suspect_rdepend = frozenset([ "dev-util/gtk-doc-am", "dev-util/intltool", "dev-util/jam", + "dev-util/pkg-config-lite", + "dev-util/pkgconf", "dev-util/pkgconfig", + "dev-util/pkgconfig-openbsd", "dev-util/scons", "dev-util/unifdef", "dev-util/yacc", @@ -497,16 +460,35 @@ suspect_rdepend = frozenset([ "sys-devel/m4", "sys-devel/pmake", "virtual/linux-sources", + "virtual/pkgconfig", "x11-misc/bdftopcf", "x11-misc/imake", ]) +suspect_virtual = { + "dev-util/pkg-config-lite":"virtual/pkgconfig", + "dev-util/pkgconf":"virtual/pkgconfig", + "dev-util/pkgconfig":"virtual/pkgconfig", + "dev-util/pkgconfig-openbsd":"virtual/pkgconfig", + "dev-libs/libusb":"virtual/libusb", + "dev-libs/libusbx":"virtual/libusb", + "dev-libs/libusb-compat":"virtual/libusb", +} + +ruby_deprecated = frozenset([ + "ruby_targets_ree18", +]) + +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \ + (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' # force refetch if the local copy creation time is older than this metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days # file.executable -no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"]) +no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"]) options, arguments = ParseArgs(sys.argv, qahelp) @@ -514,6 +496,11 @@ if options.version: print("Portage", portage.VERSION) sys.exit(0) +if options.experimental_inherit == 'y': + # This is experimental, so it's non-fatal. + qawarnings.add("inherit.missing") + repoman.checks._init(experimental_inherit=True) + # Set this to False when an extraordinary issue (generally # something other than a QA issue) makes it impossible to # commit (like if Manifest generation fails). @@ -563,14 +550,29 @@ if options.mode == 'commit' and not options.pretend and not vcs: logging.info("Not in a version controlled repository; enabling pretend mode.") options.pretend = True -# Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD. -repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \ - (repoman_settings.get('PORTDIR_OVERLAY', ''), - portage._shell_quote(portdir_overlay)) -# We have to call the config constructor again so -# that config.repositories is initialized correctly. -repoman_settings = portage.config(config_root=config_root, local_config=False, - env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY'])) +# Ensure that current repository is in the list of enabled repositories. +repodir = os.path.realpath(portdir_overlay) +try: + repoman_settings.repositories.get_repo_for_location(repodir) +except KeyError: + repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0] + layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0] + if layout_conf_data['repo-name']: + repo_name = layout_conf_data['repo-name'] + tmp_conf_file = io.StringIO(textwrap.dedent(""" + [%s] + location = %s + """) % (repo_name, portdir_overlay)) + # Ensure that the repository corresponding to $PWD overrides a + # repository of the same name referenced by the existing PORTDIR + # or PORTDIR_OVERLAY settings. + repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \ + (repoman_settings.get('PORTDIR_OVERLAY', ''), + portage._shell_quote(portdir_overlay)) + repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file]) + # We have to call the config constructor again so that attributes + # dependent on config.repositories are initialized correctly. + repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories) root = repoman_settings['EROOT'] trees = { @@ -580,10 +582,15 @@ portdb = trees[root]['porttree'].dbapi # Constrain dependency resolution to the master(s) # that are specified in layout.conf. -repodir = os.path.realpath(portdir_overlay) repo_config = repoman_settings.repositories.get_repo_for_location(repodir) portdb.porttrees = list(repo_config.eclass_db.porttrees) portdir = portdb.porttrees[0] +commit_env = os.environ.copy() +# list() is for iteration on a copy. +for repo in list(repoman_settings.repositories): + # all paths are canonical + if repo.location not in repo_config.eclass_db.porttrees: + del repoman_settings.repositories[repo.name] if repo_config.allow_provide_virtual: qawarnings.add("virtual.oldstyle") @@ -594,6 +601,15 @@ if repo_config.sign_commit: # the commit arguments. If key_id is unspecified, then it must be # configured by `git config user.signingkey key_id`. vcs_local_opts.append("--gpg-sign") + if repoman_settings.get("PORTAGE_GPG_DIR"): + # Pass GNUPGHOME to git for bug #462362. + commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"] + + # Pass GPG_TTY to git for bug #477728. + try: + commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno()) + except OSError: + pass # In order to disable manifest signatures, repos may set # "sign-manifests = false" in metadata/layout.conf. This @@ -602,6 +618,25 @@ if repo_config.sign_commit: sign_manifests = "sign" in repoman_settings.features and \ repo_config.sign_manifest +if repo_config.sign_manifest and repo_config.name == "gentoo" and \ + options.mode in ("commit",) and not sign_manifests: + msg = ("The '%s' repository has manifest signatures enabled, " + "but FEATURES=sign is currently disabled. In order to avoid this " + "warning, enable FEATURES=sign in make.conf. Alternatively, " + "repositories can disable manifest signatures by setting " + "'sign-manifests = false' in metadata/layout.conf.") % \ + (repo_config.name,) + for line in textwrap.wrap(msg, 60): + logging.warn(line) + +if sign_manifests and options.mode in ("commit",) and \ + repoman_settings.get("PORTAGE_GPG_KEY") and \ + re.match(r'^%s$' % GPG_KEY_ID_REGEX, + repoman_settings["PORTAGE_GPG_KEY"]) is None: + logging.error("PORTAGE_GPG_KEY value is invalid: %s" % + repoman_settings["PORTAGE_GPG_KEY"]) + sys.exit(1) + manifest_hashes = repo_config.manifest_hashes if manifest_hashes is None: manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS @@ -631,19 +666,6 @@ if options.mode in ("commit", "fix", "manifest"): logging.error(line) sys.exit(1) -if "commit" == options.mode and \ - repo_config.name == "gentoo" and \ - "RMD160" in manifest_hashes and \ - "RMD160" not in portage.checksum.hashorigin_map: - msg = "Please install " \ - "pycrypto or enable python's ssl USE flag in order " \ - "to enable RMD160 hash support. See bug #198398 for " \ - "more information." - prefix = bad(" * ") - for line in textwrap.wrap(msg, 70): - print(prefix + line) - sys.exit(1) - if options.echangelog is None and repo_config.update_changelog: options.echangelog = 'y' @@ -668,18 +690,9 @@ logging.debug("vcs: %s" % (vcs,)) logging.debug("repo config: %s" % (repo_config,)) logging.debug("options: %s" % (options,)) -# Generate an appropriate PORTDIR_OVERLAY value for passing into the -# profile-specific config constructor calls. -env = os.environ.copy() -env['PORTDIR'] = portdir -env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:]) - -logging.info('Setting paths:') -logging.info('PORTDIR = "' + portdir + '"') -logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY']) - # It's confusing if these warnings are displayed without the user # being told which profile they come from, so disable them. +env = os.environ.copy() env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' categories = [] @@ -703,7 +716,7 @@ repolevel = len(reposplit) # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting. # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating. # this check ensures that repoman knows where it is, and the manifest recommit is at least possible. -if options.mode == 'commit' and repolevel not in [1,2,3]: +if options.mode == 'commit' and repolevel not in [1, 2, 3]: print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.") print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.") print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.") @@ -716,10 +729,76 @@ if repolevel == 1: startdir = repodir else: startdir = normalize_path(mydir) - startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:]) + startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:]) def caterror(mycat): - err(mycat+" is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.") + err(mycat + " is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.") + +def repoman_getstatusoutput(cmd): + """ + Implements an interface similar to getstatusoutput(), but with + customized unicode handling (see bug #310789) and without the shell. + """ + args = portage.util.shlex_split(cmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + encoding = _encodings['fs'] + args = [_unicode_encode(x, + encoding=encoding, errors='strict') for x in args] + proc = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = portage._unicode_decode(proc.communicate()[0], + encoding=encoding, errors='strict') + if output and output[-1] == "\n": + # getstatusoutput strips one newline + output = output[:-1] + return (proc.wait(), output) + +class repoman_popen(portage.proxy.objectproxy.ObjectProxy): + """ + Implements an interface similar to os.popen(), but with customized + unicode handling (see bug #310789) and without the shell. + """ + + __slots__ = ('_proc', '_stdout') + + def __init__(self, cmd): + args = portage.util.shlex_split(cmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + encoding = _encodings['fs'] + args = [_unicode_encode(x, + encoding=encoding, errors='strict') for x in args] + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + object.__setattr__(self, '_proc', proc) + object.__setattr__(self, '_stdout', + codecs.getreader(encoding)(proc.stdout, 'strict')) + + def _get_target(self): + return object.__getattribute__(self, '_stdout') + + __enter__ = _get_target + + def __exit__(self, exc_type, exc_value, traceback): + proc = object.__getattribute__(self, '_proc') + proc.wait() + proc.stdout.close() class ProfileDesc(object): __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',) @@ -797,18 +876,18 @@ for path in portdb.porttrees: continue if len(arch) != 3: err("wrong format: \"" + bad(x.strip()) + "\" in " + \ - desc_path + " line %d" % (i+1, )) + desc_path + " line %d" % (i + 1, )) elif arch[0] not in kwlist: err("invalid arch: \"" + bad(arch[0]) + "\" in " + \ - desc_path + " line %d" % (i+1, )) + desc_path + " line %d" % (i + 1, )) elif arch[2] not in valid_profile_types: err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \ - desc_path + " line %d" % (i+1, )) + desc_path + " line %d" % (i + 1, )) profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path) if not os.path.isdir(profile_desc.abs_path): logging.error( "Invalid %s profile (%s) for arch %s in %s line %d", - arch[2], arch[1], arch[0], desc_path, i+1) + arch[2], arch[1], arch[0], desc_path, i + 1) continue if os.path.exists( os.path.join(profile_desc.abs_path, 'deprecated')): @@ -855,11 +934,16 @@ for x in repoman_settings.archlist(): if x[0] == "~": continue if x not in profiles: - print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")) + print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc.")) print(red("You need to either \"cvs update\" your profiles dir or follow this")) - print(red("up with the "+x+" team.")) + print(red("up with the " + x + " team.")) print() +liclist_deprecated = set() +if "DEPRECATED" in repoman_settings._license_manager._license_groups: + liclist_deprecated.update( + repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"])) + if not liclist: logging.fatal("Couldn't find licenses?") sys.exit(1) @@ -872,34 +956,34 @@ if not uselist: logging.fatal("Couldn't find use.desc?") sys.exit(1) -scanlist=[] -if repolevel==2: - #we are inside a category directory - catdir=reposplit[-1] +scanlist = [] +if repolevel == 2: + # we are inside a category directory + catdir = reposplit[-1] if catdir not in categories: caterror(catdir) - mydirlist=os.listdir(startdir) + mydirlist = os.listdir(startdir) for x in mydirlist: if x == "CVS" or x.startswith("."): continue - if os.path.isdir(startdir+"/"+x): - scanlist.append(catdir+"/"+x) + if os.path.isdir(startdir + "/" + x): + scanlist.append(catdir + "/" + x) repo_subdir = catdir + os.sep -elif repolevel==1: +elif repolevel == 1: for x in categories: - if not os.path.isdir(startdir+"/"+x): + if not os.path.isdir(startdir + "/" + x): continue - for y in os.listdir(startdir+"/"+x): + for y in os.listdir(startdir + "/" + x): if y == "CVS" or y.startswith("."): continue - if os.path.isdir(startdir+"/"+x+"/"+y): - scanlist.append(x+"/"+y) + if os.path.isdir(startdir + "/" + x + "/" + y): + scanlist.append(x + "/" + y) repo_subdir = "" -elif repolevel==3: +elif repolevel == 3: catdir = reposplit[-2] if catdir not in categories: caterror(catdir) - scanlist.append(catdir+"/"+reposplit[-1]) + scanlist.append(catdir + "/" + reposplit[-1]) repo_subdir = scanlist[-1] + os.sep else: msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \ @@ -931,7 +1015,7 @@ def vcs_files_to_cps(vcs_file_iter): if category in categories: for filename in vcs_file_iter: f_split = filename.split(os.sep) - # ['.', pn,...] + # ['.', pn, ...] if len(f_split) > 2: modified_cps.append(category + "/" + f_split[1]) @@ -939,7 +1023,7 @@ def vcs_files_to_cps(vcs_file_iter): # repolevel == 1 for filename in vcs_file_iter: f_split = filename.split(os.sep) - # ['.', category, pn,...] + # ['.', category, pn, ...] if len(f_split) > 3 and f_split[1] in categories: modified_cps.append("/".join(f_split[1:3])) @@ -947,12 +1031,12 @@ def vcs_files_to_cps(vcs_file_iter): def git_supports_gpg_sign(): status, cmd_output = \ - subprocess_getstatusoutput("git --version") + repoman_getstatusoutput("git --version") cmd_output = cmd_output.split() if cmd_output: version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1]) if version is not None: - version = [int(x) for x in version.groups()[1:]] + version = [int(x) for x in version.groups()] if version[0] > 1 or \ (version[0] == 1 and version[1] > 7) or \ (version[0] == 1 and version[1] == 7 and version[2] >= 9): @@ -981,47 +1065,16 @@ def dev_keywords(profiles): dev_keywords = dev_keywords(profiles) -stats={} -fails={} - -# provided by the desktop-file-utils package -desktop_file_validate = find_binary("desktop-file-validate") -desktop_pattern = re.compile(r'.*\.desktop$') +stats = {} +fails = {} for x in qacats: - stats[x]=0 - fails[x]=[] + stats[x] = 0 + fails[x] = [] xmllint_capable = False metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd') -def parsedate(s): - """Parse a RFC 822 date and time string. - This is required for python3 compatibility, since the - rfc822.parsedate() function is not available.""" - - s_split = [] - for x in s.upper().split(): - for y in x.split(','): - if y: - s_split.append(y) - - if len(s_split) != 6: - return None - - # %a, %d %b %Y %H:%M:%S %Z - a, d, b, Y, H_M_S, Z = s_split - - # Convert month to integer, since strptime %w is locale-dependent. - month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6, - 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12} - m = month_map.get(b) - if m is None: - return None - m = str(m).rjust(2, '0') - - return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S') - def fetch_metadata_dtd(): """ Fetch metadata.dtd if it doesn't exist or the ctime is older than @@ -1050,45 +1103,40 @@ def fetch_metadata_dtd(): print(green("***") + " the local copy of metadata.dtd " + \ "needs to be refetched, doing that now") print() + parsed_url = urlparse(metadata_dtd_uri) + setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() + fcmd = repoman_settings.get(setting) + if not fcmd: + fcmd = repoman_settings.get('FETCHCOMMAND') + if not fcmd: + logging.error("FETCHCOMMAND is unset") + return False + + destdir = repoman_settings["DISTDIR"] + fd, metadata_dtd_tmp = tempfile.mkstemp( + prefix='metadata.dtd.', dir=destdir) + os.close(fd) + try: - url_f = urllib_request_urlopen(metadata_dtd_uri) - msg_info = url_f.info() - last_modified = msg_info.get('last-modified') - if last_modified is not None: - last_modified = parsedate(last_modified) - if last_modified is not None: - last_modified = calendar.timegm(last_modified) - - metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid()) - try: - local_f = open(metadata_dtd_tmp, mode='wb') - local_f.write(url_f.read()) - local_f.close() - if last_modified is not None: - try: - os.utime(metadata_dtd_tmp, - (int(last_modified), int(last_modified))) - except OSError: - # This fails on some odd non-unix-like filesystems. - # We don't really need the mtime to be preserved - # anyway here (currently we use ctime to trigger - # fetch), so just ignore it. - pass - os.rename(metadata_dtd_tmp, metadata_dtd) - finally: - try: - os.unlink(metadata_dtd_tmp) - except OSError: - pass + if not portage.getbinpkg.file_get(metadata_dtd_uri, + destdir, fcmd=fcmd, + filename=os.path.basename(metadata_dtd_tmp)): + logging.error("failed to fetch metadata.dtd from '%s'" % + metadata_dtd_uri) + return False - url_f.close() + try: + portage.util.apply_secpass_permissions(metadata_dtd_tmp, + gid=portage.data.portage_gid, mode=0o664, mask=0o2) + except portage.exception.PortageException: + pass - except EnvironmentError as e: - print() - print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri) - print(red("!!!")+" exception '%s' though." % (e,)) - print(red("!!!")+" fetching new metadata.dtd failed, aborting") - return False + os.rename(metadata_dtd_tmp, metadata_dtd) + finally: + try: + os.unlink(metadata_dtd_tmp) + except OSError: + pass return True @@ -1096,14 +1144,14 @@ if options.mode == "manifest": pass elif not find_binary('xmllint'): print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - if options.xml_parse or repolevel==3: + if options.xml_parse or repolevel == 3: print(red("!!!")+" sorry, xmllint is needed. failing\n") sys.exit(1) else: if not fetch_metadata_dtd(): sys.exit(1) - #this can be problematic if xmllint changes their output - xmllint_capable=True + # this can be problematic if xmllint changes their output + xmllint_capable = True if options.mode == 'commit' and vcs: utilities.detect_vcs_conflicts(options, vcs) @@ -1130,45 +1178,46 @@ if vcs == "cvs": myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./") elif vcs == "svn": - with os.popen("svn status") as f: + with repoman_popen("svn status") as f: svnstatus = f.readlines() - mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ] - mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ] + mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"] + mynew = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")] if options.if_modified == "y": - myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] + myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] elif vcs == "git": - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=M HEAD") as f: mychanged = f.readlines() mychanged = ["./" + elem[:-1] for elem in mychanged] - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=A HEAD") as f: mynew = f.readlines() mynew = ["./" + elem[:-1] for elem in mynew] if options.if_modified == "y": - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=D HEAD") as f: myremoved = f.readlines() myremoved = ["./" + elem[:-1] for elem in myremoved] elif vcs == "bzr": - with os.popen("bzr status -S .") as f: + with repoman_popen("bzr status -S .") as f: bzrstatus = f.readlines() - mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ] - mynew = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ] + mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"] + mynew = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")] if options.if_modified == "y": - myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ] + myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")] elif vcs == "hg": - with os.popen("hg status --no-status --modified .") as f: + with repoman_popen("hg status --no-status --modified .") as f: mychanged = f.readlines() mychanged = ["./" + elem.rstrip() for elem in mychanged] - mynew = os.popen("hg status --no-status --added .").readlines() + with repoman_popen("hg status --no-status --added .") as f: + mynew = f.readlines() mynew = ["./" + elem.rstrip() for elem in mynew] if options.if_modified == "y": - with os.popen("hg status --no-status --removed .") as f: + with repoman_popen("hg status --no-status --removed .") as f: myremoved = f.readlines() myremoved = ["./" + elem.rstrip() for elem in myremoved] @@ -1190,10 +1239,15 @@ dofail = 0 # NOTE: match-all caches are not shared due to potential # differences between profiles in _get_implicit_iuse. -arch_caches={} +arch_caches = {} arch_xmatch_caches = {} shared_xmatch_caches = {"cp-list":{}} +include_arches = None +if options.include_arches: + include_arches = set() + include_arches.update(*[x.split() for x in options.include_arches]) + # Disable the "ebuild.notadded" check when not in commit mode and # running `svn status` in every package dir will be too expensive. @@ -1201,12 +1255,37 @@ check_ebuild_notadded = not \ (vcs == "svn" and repolevel < 3 and options.mode != "commit") # Build a regex from thirdpartymirrors for the SRC_URI.mirror check. -thirdpartymirrors = [] -for v in repoman_settings.thirdpartymirrors().values(): +thirdpartymirrors = {} +for k, v in repoman_settings.thirdpartymirrors().items(): for v in v: if not v.endswith("/"): v += "/" - thirdpartymirrors.append(v) + thirdpartymirrors[v] = k + +class _XMLParser(xml.etree.ElementTree.XMLParser): + + def __init__(self, data, **kwargs): + xml.etree.ElementTree.XMLParser.__init__(self, **kwargs) + self._portage_data = data + if hasattr(self, 'parser'): + self._base_XmlDeclHandler = self.parser.XmlDeclHandler + self.parser.XmlDeclHandler = self._portage_XmlDeclHandler + self._base_StartDoctypeDeclHandler = \ + self.parser.StartDoctypeDeclHandler + self.parser.StartDoctypeDeclHandler = \ + self._portage_StartDoctypeDeclHandler + + def _portage_XmlDeclHandler(self, version, encoding, standalone): + if self._base_XmlDeclHandler is not None: + self._base_XmlDeclHandler(version, encoding, standalone) + self._portage_data["XML_DECLARATION"] = (version, encoding, standalone) + + def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, + has_internal_subset): + if self._base_StartDoctypeDeclHandler is not None: + self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId, + has_internal_subset) + self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId) class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): """ @@ -1231,13 +1310,13 @@ if options.if_modified == "y": chain(mychanged, mynew, myremoved))) for x in effective_scanlist: - #ebuilds and digests added to cvs respectively. + # ebuilds and digests added to cvs respectively. logging.info("checking package %s" % x) # save memory by discarding xmatch caches from previous package(s) arch_xmatch_caches.clear() - eadded=[] - catdir,pkgdir=x.split("/") - checkdir=repodir+"/"+x + eadded = [] + catdir, pkgdir = x.split("/") + checkdir = repodir + "/" + x checkdir_relative = "" if repolevel < 3: checkdir_relative = os.path.join(pkgdir, checkdir_relative) @@ -1319,15 +1398,15 @@ for x in effective_scanlist: if options.mode == 'manifest-check': continue - checkdirlist=os.listdir(checkdir) - ebuildlist=[] + checkdirlist = os.listdir(checkdir) + ebuildlist = [] pkgs = {} allvalid = True for y in checkdirlist: if (y in no_exec or y.endswith(".ebuild")) and \ - stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111: - stats["file.executable"] += 1 - fails["file.executable"].append(os.path.join(checkdir, y)) + stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111: + stats["file.executable"] += 1 + fails["file.executable"].append(os.path.join(checkdir, y)) if y.endswith(".ebuild"): pf = y[:-7] ebuildlist.append(pf) @@ -1368,19 +1447,19 @@ for x in effective_scanlist: ebuildlist = [pkg.pf for pkg in ebuildlist] for y in checkdirlist: - m = disallowed_filename_chars_re.search(y.strip(os.sep)) - if m is not None: + index = repo_config.find_invalid_path_char(y) + if index != -1: y_relative = os.path.join(checkdir_relative, y) if vcs is not None and not vcs_new_changed(y_relative): # If the file isn't in the VCS new or changed set, then # assume that it's an irrelevant temporary file (Manifest # entries are not generated for file names containing # prohibited characters). See bug #406877. - m = None - if m is not None: + index = -1 + if index != -1: stats["file.name"] += 1 fails["file.name"].append("%s/%s: char '%s'" % \ - (checkdir, y, m.group(0))) + (checkdir, y, y[index])) if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")): continue @@ -1391,7 +1470,7 @@ for x in effective_scanlist: encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content']) for l in f: - line +=1 + line += 1 except UnicodeDecodeError as ue: stats["file.UTF8"] += 1 s = ue.object[:ue.start] @@ -1406,10 +1485,10 @@ for x in effective_scanlist: if vcs in ("git", "hg") and check_ebuild_notadded: if vcs == "git": - myf = os.popen("git ls-files --others %s" % \ + myf = repoman_popen("git ls-files --others %s" % \ (portage._shell_quote(checkdir_relative),)) if vcs == "hg": - myf = os.popen("hg status --no-status --unknown %s" % \ + myf = repoman_popen("hg status --no-status --unknown %s" % \ (portage._shell_quote(checkdir_relative),)) for l in myf: if l[:-1][-7:] == ".ebuild": @@ -1421,21 +1500,23 @@ for x in effective_scanlist: if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded: try: if vcs == "cvs": - myf=open(checkdir+"/CVS/Entries","r") + myf = open(checkdir + "/CVS/Entries", "r") if vcs == "svn": - myf = os.popen("svn status --depth=files --verbose " + checkdir) + myf = repoman_popen("svn status --depth=files --verbose " + + portage._shell_quote(checkdir)) if vcs == "bzr": - myf = os.popen("bzr ls -v --kind=file " + checkdir) + myf = repoman_popen("bzr ls -v --kind=file " + + portage._shell_quote(checkdir)) myl = myf.readlines() myf.close() for l in myl: if vcs == "cvs": - if l[0]!="/": + if l[0] != "/": continue - splitl=l[1:].split("/") + splitl = l[1:].split("/") if not len(splitl): continue - if splitl[0][-7:]==".ebuild": + if splitl[0][-7:] == ".ebuild": eadded.append(splitl[0][:-7]) if vcs == "svn": if l[:1] == "?": @@ -1453,8 +1534,9 @@ for x in effective_scanlist: if l[-7:] == ".ebuild": eadded.append(os.path.basename(l[:-7])) if vcs == "svn": - myf = os.popen("svn status " + checkdir) - myl=myf.readlines() + myf = repoman_popen("svn status " + + portage._shell_quote(checkdir)) + myl = myf.readlines() myf.close() for l in myl: if l[0] == "A": @@ -1464,7 +1546,7 @@ for x in effective_scanlist: except IOError: if vcs == "cvs": stats["CVS/Entries.IO_error"] += 1 - fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries") + fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries") else: raise continue @@ -1472,7 +1554,7 @@ for x in effective_scanlist: mf = repoman_settings.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(checkdir))) mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"]) - mydigests=mf.getTypeDigests("DIST") + mydigests = mf.getTypeDigests("DIST") fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb) myfiles_all = [] @@ -1488,7 +1570,7 @@ for x in effective_scanlist: # This will be reported as an "ebuild.syntax" error. pass else: - stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1 + stats["SRC_URI.syntax"] += 1 fails["SRC_URI.syntax"].append( "%s.ebuild SRC_URI: %s" % (mykey, e)) del fetchlist_dict @@ -1502,15 +1584,15 @@ for x in effective_scanlist: for entry in mydigests: if entry not in myfiles_all: stats["digest.unused"] += 1 - fails["digest.unused"].append(checkdir+"::"+entry) + fails["digest.unused"].append(checkdir + "::" + entry) for entry in myfiles_all: if entry not in mydigests: stats["digest.missing"] += 1 - fails["digest.missing"].append(checkdir+"::"+entry) + fails["digest.missing"].append(checkdir + "::" + entry) del myfiles_all - if os.path.exists(checkdir+"/files"): - filesdirlist=os.listdir(checkdir+"/files") + if os.path.exists(checkdir + "/files"): + filesdirlist = os.listdir(checkdir + "/files") # recurse through files directory # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory. @@ -1530,77 +1612,110 @@ for x in effective_scanlist: # !!! VCS "portability" alert! Need some function isVcsDir() or alike !!! if y == "CVS" or y == ".svn": continue - for z in os.listdir(checkdir+"/files/"+y): + for z in os.listdir(checkdir + "/files/" + y): if z == "CVS" or z == ".svn": continue - filesdirlist.append(y+"/"+z) + filesdirlist.append(y + "/" + z) # Current policy is no files over 20 KiB, these are the checks. File size between # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error. elif mystat.st_size > 61440: stats["file.size.fatal"] += 1 - fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y) + fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y) elif mystat.st_size > 20480: stats["file.size"] += 1 - fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y) + fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y) - m = disallowed_filename_chars_re.search( - os.path.basename(y.rstrip(os.sep))) - if m is not None: + index = repo_config.find_invalid_path_char(y) + if index != -1: y_relative = os.path.join(checkdir_relative, "files", y) if vcs is not None and not vcs_new_changed(y_relative): # If the file isn't in the VCS new or changed set, then # assume that it's an irrelevant temporary file (Manifest # entries are not generated for file names containing # prohibited characters). See bug #406877. - m = None - if m is not None: + index = -1 + if index != -1: stats["file.name"] += 1 fails["file.name"].append("%s/files/%s: char '%s'" % \ - (checkdir, y, m.group(0))) - - if desktop_file_validate and desktop_pattern.match(y): - cmd_output = validate_desktop_entry(full_path) - if cmd_output: - # Note: in the future we may want to grab the - # warnings in addition to the errors. We're - # just doing errors now since we don't want - # to generate too much noise at first. - error_re = re.compile(r'.*\s*error:\s*(.*)') - for line in cmd_output: - error_match = error_re.match(line) - if error_match is None: - continue - stats["desktop.invalid"] += 1 - fails["desktop.invalid"].append( - relative_path + ': %s' % error_match.group(1)) - + (checkdir, y, y[index])) del mydigests if check_changelog and "ChangeLog" not in checkdirlist: - stats["changelog.missing"]+=1 - fails["changelog.missing"].append(x+"/ChangeLog") - + stats["changelog.missing"] += 1 + fails["changelog.missing"].append(x + "/ChangeLog") + musedict = {} - #metadata.xml file check + # metadata.xml file check if "metadata.xml" not in checkdirlist: - stats["metadata.missing"]+=1 - fails["metadata.missing"].append(x+"/metadata.xml") - #metadata.xml parse check + stats["metadata.missing"] += 1 + fails["metadata.missing"].append(x + "/metadata.xml") + # metadata.xml parse check else: metadata_bad = False + xml_info = {} + xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) # read metadata.xml into memory try: _metadata_xml = xml.etree.ElementTree.parse( - os.path.join(checkdir, "metadata.xml"), - parser=xml.etree.ElementTree.XMLParser( - target=_MetadataTreeBuilder())) + _unicode_encode(os.path.join(checkdir, "metadata.xml"), + encoding=_encodings['fs'], errors='strict'), + parser=xml_parser) except (ExpatError, SyntaxError, EnvironmentError) as e: metadata_bad = True stats["metadata.bad"] += 1 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e)) del e else: + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x2070000 or \ + (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000): + # doctype is not parsed with python 2.6 or 3.1 + pass + else: + if "XML_DECLARATION" not in xml_info: + stats["metadata.bad"] += 1 + fails["metadata.bad"].append("%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (x, metadata_xml_declaration)) + else: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + stats["metadata.bad"] += 1 + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding + fails["metadata.bad"].append("%s/metadata.xml: " + "xml declaration encoding should be '%s', %s" % + (x, metadata_xml_encoding, encoding_problem)) + + if "DOCTYPE" not in xml_info: + metadata_bad = True + stats["metadata.bad"] += 1 + fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, + "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + stats["metadata.bad"] += 1 + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system + fails["metadata.bad"].append("%s/metadata.xml: " + "DOCTYPE: SYSTEM should refer to '%s', %s" % + (x, metadata_dtd_uri, system_problem)) + + if doctype_name != metadata_doctype_name: + stats["metadata.bad"] += 1 + fails["metadata.bad"].append("%s/metadata.xml: " + "DOCTYPE: name should be '%s', not '%s'" % + (x, metadata_doctype_name, doctype_name)) + # load USE flags from metadata.xml try: musedict = utilities.parse_metadata_use(_metadata_xml) @@ -1608,6 +1723,22 @@ for x in effective_scanlist: metadata_bad = True stats["metadata.bad"] += 1 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e)) + else: + for atom in chain(*musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + stats["metadata.bad"] += 1 + fails["metadata.bad"].append( + "%s/metadata.xml: Invalid atom: %s" % (x, e)) + else: + if atom.cp != x: + stats["metadata.bad"] += 1 + fails["metadata.bad"].append( + ("%s/metadata.xml: Atom contains " + "unexpected cat/pn: %s") % (x, atom)) # Run other metadata.xml checkers try: @@ -1618,19 +1749,20 @@ for x in effective_scanlist: fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e)) del e - #Only carry out if in package directory or check forced + # Only carry out if in package directory or check forced if xmllint_capable and not metadata_bad: # xmlint can produce garbage output even on success, so only dump # the ouput when it fails. - st, out = subprocess_getstatusoutput( - "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \ - (metadata_dtd, os.path.join(checkdir, "metadata.xml"))) + st, out = repoman_getstatusoutput( + "xmllint --nonet --noout --dtdvalid %s %s" % \ + (portage._shell_quote(metadata_dtd), + portage._shell_quote(os.path.join(checkdir, "metadata.xml")))) if st != os.EX_OK: print(red("!!!") + " metadata.xml is invalid:") for z in out.splitlines(): - print(red("!!! ")+z) - stats["metadata.bad"]+=1 - fails["metadata.bad"].append(x+"/metadata.xml") + print(red("!!! ") + z) + stats["metadata.bad"] += 1 + fails["metadata.bad"].append(x + "/metadata.xml") del metadata_bad muselist = frozenset(musedict) @@ -1656,20 +1788,20 @@ for x in effective_scanlist: fails['changelog.ebuildadded'].append(relative_path) if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded: - #ebuild not added to vcs - stats["ebuild.notadded"]=stats["ebuild.notadded"]+1 - fails["ebuild.notadded"].append(x+"/"+y+".ebuild") - myesplit=portage.pkgsplit(y) + # ebuild not added to vcs + stats["ebuild.notadded"] += 1 + fails["ebuild.notadded"].append(x + "/" + y + ".ebuild") + myesplit = portage.pkgsplit(y) if myesplit is None or myesplit[0] != x.split("/")[-1] \ or pv_toolong_re.search(myesplit[1]) \ or pv_toolong_re.search(myesplit[2]): - stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1 - fails["ebuild.invalidname"].append(x+"/"+y+".ebuild") + stats["ebuild.invalidname"] += 1 + fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild") continue - elif myesplit[0]!=pkgdir: - print(pkgdir,myesplit[0]) - stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1 - fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild") + elif myesplit[0] != pkgdir: + print(pkgdir, myesplit[0]) + stats["ebuild.namenomatch"] += 1 + fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild") continue pkg = pkgs[y] @@ -1678,15 +1810,25 @@ for x in effective_scanlist: allvalid = False for k, msgs in pkg.invalid.items(): for msg in msgs: - stats[k] = stats[k] + 1 - fails[k].append("%s %s" % (relative_path, msg)) + stats[k] += 1 + fails[k].append("%s: %s" % (relative_path, msg)) continue - myaux = pkg.metadata + myaux = pkg._metadata eapi = myaux["EAPI"] inherited = pkg.inherited live_ebuild = live_eclasses.intersection(inherited) + if repo_config.eapi_is_banned(eapi): + stats["repo.eapi.banned"] += 1 + fails["repo.eapi.banned"].append( + "%s: %s" % (relative_path, eapi)) + + elif repo_config.eapi_is_deprecated(eapi): + stats["repo.eapi.deprecated"] += 1 + fails["repo.eapi.deprecated"].append( + "%s: %s" % (relative_path, eapi)) + for k, v in myaux.items(): if not isinstance(v, basestring): continue @@ -1703,20 +1845,21 @@ for x in effective_scanlist: for uri in portage.dep.use_reduce( \ myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True): contains_mirror = False - for mirror in thirdpartymirrors: + for mirror, mirror_alias in thirdpartymirrors.items(): if uri.startswith(mirror): contains_mirror = True break if not contains_mirror: continue + new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):]) stats["SRC_URI.mirror"] += 1 fails["SRC_URI.mirror"].append( - "%s: '%s' found in thirdpartymirrors" % \ - (relative_path, mirror)) + "%s: '%s' found in thirdpartymirrors, use '%s'" % \ + (relative_path, mirror, new_uri)) if myaux.get("PROVIDE"): - stats["virtual.oldstyle"]+=1 + stats["virtual.oldstyle"] += 1 fails["virtual.oldstyle"].append(relative_path) for pos, missing_var in enumerate(missingvars): @@ -1726,15 +1869,15 @@ for x in effective_scanlist: continue if live_ebuild and missing_var == "KEYWORDS": continue - myqakey=missingvars[pos]+".missing" - stats[myqakey]=stats[myqakey]+1 - fails[myqakey].append(x+"/"+y+".ebuild") + myqakey = missingvars[pos] + ".missing" + stats[myqakey] += 1 + fails[myqakey].append(x + "/" + y + ".ebuild") if catdir == "virtual": for var in ("HOMEPAGE", "LICENSE"): if myaux.get(var): myqakey = var + ".virtual" - stats[myqakey] = stats[myqakey] + 1 + stats[myqakey] += 1 fails[myqakey].append(relative_path) # 14 is the length of DESCRIPTION="" @@ -1751,7 +1894,7 @@ for x in effective_scanlist: not keyword.startswith("-"): stable_keywords.append(keyword) if stable_keywords: - if ebuild_path in new_ebuilds: + if ebuild_path in new_ebuilds and catdir != "virtual": stable_keywords.sort() stats["KEYWORDS.stable"] += 1 fails["KEYWORDS.stable"].append( @@ -1761,10 +1904,10 @@ for x in effective_scanlist: ebuild_archs = set(kw.lstrip("~") for kw in keywords \ if not kw.startswith("-")) - previous_keywords = slot_keywords.get(myaux["SLOT"]) + previous_keywords = slot_keywords.get(pkg.slot) if previous_keywords is None: - slot_keywords[myaux["SLOT"]] = set() - elif ebuild_archs and not live_ebuild: + slot_keywords[pkg.slot] = set() + elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild: dropped_keywords = previous_keywords.difference(ebuild_archs) if dropped_keywords: stats["KEYWORDS.dropped"] += 1 @@ -1772,7 +1915,7 @@ for x in effective_scanlist: relative_path + ": %s" % \ " ".join(sorted(dropped_keywords))) - slot_keywords[myaux["SLOT"]].update(ebuild_archs) + slot_keywords[pkg.slot].update(ebuild_archs) # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics if "-*" in keywords: @@ -1784,7 +1927,7 @@ for x in effective_scanlist: haskeyword = True if not haskeyword: stats["KEYWORDS.stupid"] += 1 - fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild") + fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild") """ Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should @@ -1812,37 +1955,53 @@ for x in effective_scanlist: arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"], repoman_settings["ACCEPT_KEYWORDS"].split()]] else: - arches=[] - for keyword in myaux["KEYWORDS"].split(): - if (keyword[0]=="-"): + arches = set() + for keyword in keywords: + if keyword[0] == "-": continue - elif (keyword[0]=="~"): - arches.append([keyword, keyword[1:], [keyword[1:], keyword]]) + elif keyword[0] == "~": + arch = keyword[1:] + if arch == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add((keyword, expanded_arch, + (expanded_arch, "~" + expanded_arch))) + else: + arches.add((keyword, arch, (arch, keyword))) else: - arches.append([keyword, keyword, [keyword]]) + if keyword == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add((keyword, expanded_arch, + (expanded_arch,))) + else: + arches.add((keyword, keyword, (keyword,))) if not arches: # Use an empty profile for checking dependencies of # packages that have empty KEYWORDS. - arches.append(['**', '**', ['**']]) + arches.add(('**', '**', ('**',))) unknown_pkgs = set() baddepsyntax = False badlicsyntax = False badprovsyntax = False - catpkg = catdir+"/"+y + catpkg = catdir + "/" + y inherited_java_eclass = "java-pkg-2" in inherited or \ "java-pkg-opt-2" in inherited inherited_wxwidgets_eclass = "wxwidgets" in inherited operator_tokens = set(["||", "(", ")"]) type_list, badsyntax = [], [] - for mytype in ("DEPEND", "RDEPEND", "PDEPEND", - "LICENSE", "PROPERTIES", "PROVIDE"): + for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): mydepstr = myaux[mytype] + buildtime = mytype in Package._buildtime_keys + runtime = mytype in Package._runtime_keys token_class = None - if mytype in ("DEPEND", "RDEPEND", "PDEPEND"): - token_class=portage.dep.Atom + if mytype.endswith("DEPEND"): + token_class = portage.dep.Atom try: atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \ @@ -1851,8 +2010,8 @@ for x in effective_scanlist: atoms = None badsyntax.append(str(e)) - if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"): - if mytype in ("RDEPEND", "PDEPEND") and \ + if atoms and mytype.endswith("DEPEND"): + if runtime and \ "test?" in mydepstr.split(): stats[mytype + '.suspect'] += 1 fails[mytype + '.suspect'].append(relative_path + \ @@ -1872,21 +2031,30 @@ for x in effective_scanlist: is_blocker = atom.blocker - if mytype == "DEPEND" and \ + if catdir != "virtual": + if not is_blocker and \ + atom.cp in suspect_virtual: + stats['virtual.suspect'] += 1 + fails['virtual.suspect'].append( + relative_path + + ": %s: consider using '%s' instead of '%s'" % + (mytype, suspect_virtual[atom.cp], atom)) + + if buildtime and \ not is_blocker and \ not inherited_java_eclass and \ atom.cp == "virtual/jdk": stats['java.eclassesnotused'] += 1 fails['java.eclassesnotused'].append(relative_path) - elif mytype == "DEPEND" and \ + elif buildtime and \ not is_blocker and \ not inherited_wxwidgets_eclass and \ atom.cp == "x11-libs/wxGTK": stats['wxwidgets.eclassnotused'] += 1 fails['wxwidgets.eclassnotused'].append( - relative_path + ": DEPENDs on x11-libs/wxGTK" - " without inheriting wxwidgets.eclass") - elif mytype in ("PDEPEND", "RDEPEND"): + (relative_path + ": %ss on x11-libs/wxGTK" + " without inheriting wxwidgets.eclass") % mytype) + elif runtime: if not is_blocker and \ atom.cp in suspect_rdepend: stats[mytype + '.suspect'] += 1 @@ -1895,21 +2063,26 @@ for x in effective_scanlist: if atom.operator == "~" and \ portage.versions.catpkgsplit(atom.cpv)[3] != "r0": - stats[mytype + '.badtilde'] += 1 - fails[mytype + '.badtilde'].append( + qacat = 'dependency.badtilde' + stats[qacat] += 1 + fails[qacat].append( (relative_path + ": %s uses the ~ operator" " with a non-zero revision:" + \ " '%s'") % (mytype, atom)) type_list.extend([mytype] * (len(badsyntax) - len(type_list))) - for m,b in zip(type_list, badsyntax): - stats[m+".syntax"] += 1 - fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b) + for m, b in zip(type_list, badsyntax): + if m.endswith("DEPEND"): + qacat = "dependency.syntax" + else: + qacat = m + ".syntax" + stats[qacat] += 1 + fails[qacat].append("%s: %s: %s" % (relative_path, m, b)) badlicsyntax = len([z for z in type_list if z == "LICENSE"]) badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) - baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax + baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax badlicsyntax = badlicsyntax > 0 badprovsyntax = badprovsyntax > 0 @@ -1925,7 +2098,7 @@ for x in effective_scanlist: myuse.append(flag_name) # uselist checks - metadata - for mypos in range(len(myuse)-1,-1,-1): + for mypos in range(len(myuse)-1, -1, -1): if myuse[mypos] and (myuse[mypos] in muselist): del myuse[mypos] @@ -1938,8 +2111,17 @@ for x in effective_scanlist: " '%s'") % (eapi, myflag)) for mypos in range(len(myuse)): - stats["IUSE.invalid"]=stats["IUSE.invalid"]+1 - fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos]) + stats["IUSE.invalid"] += 1 + fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos]) + + # Check for outdated RUBY targets + if "ruby-ng" in inherited or "ruby-fakegem" in inherited or "ruby" in inherited: + ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated) + if ruby_intersection: + for myruby in ruby_intersection: + stats["IUSE.rubydeprecated"] += 1 + fails["IUSE.rubydeprecated"].append( + (relative_path + ": Deprecated ruby target: %s") % myruby) # license checks if not badlicsyntax: @@ -1952,10 +2134,13 @@ for x in effective_scanlist: # Need to check for "||" manually as no portage # function will remove it without removing values. if lic not in liclist and lic != "||": - stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1 - fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic) + stats["LICENSE.invalid"] += 1 + fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic) + elif lic in liclist_deprecated: + stats["LICENSE.deprecated"] += 1 + fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic)) - #keyword checks + # keyword checks myuse = myaux["KEYWORDS"].split() for mykey in myuse: if mykey not in ("-*", "*", "~*"): @@ -1966,17 +2151,17 @@ for x in effective_scanlist: myskey = myskey[1:] if myskey not in kwlist: stats["KEYWORDS.invalid"] += 1 - fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey) + fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey) elif myskey not in profiles: stats["KEYWORDS.invalid"] += 1 - fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey) + fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey) - #restrict checks + # restrict checks myrestrict = None try: myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1 + stats["RESTRICT.syntax"] += 1 fails["RESTRICT.syntax"].append( "%s: RESTRICT: %s" % (relative_path, e)) del e @@ -1986,8 +2171,8 @@ for x in effective_scanlist: if mybadrestrict: stats["RESTRICT.invalid"] += len(mybadrestrict) for mybad in mybadrestrict: - fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad) - #REQUIRED_USE check + fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad) + # REQUIRED_USE check required_use = myaux["REQUIRED_USE"] if required_use: if not eapi_has_required_use(eapi): @@ -1997,9 +2182,9 @@ for x in effective_scanlist: " not supported with EAPI='%s'" % (eapi,)) try: portage.dep.check_required_use(required_use, (), - pkg.iuse.is_valid_flag) + pkg.iuse.is_valid_flag, eapi=eapi) except portage.exception.InvalidDependString as e: - stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1 + stats["REQUIRED_USE.syntax"] += 1 fails["REQUIRED_USE.syntax"].append( "%s: REQUIRED_USE: %s" % (relative_path, e)) del e @@ -2032,127 +2217,154 @@ for x in effective_scanlist: # user is intent on forcing the commit anyway. continue - for keyword,arch,groups in arches: - + relevant_profiles = [] + for keyword, arch, groups in arches: if arch not in profiles: # A missing profile will create an error further down # during the KEYWORDS verification. continue - - for prof in profiles[arch]: - if prof.status not in ("stable", "dev") or \ - prof.status == "dev" and not options.include_dev: + if include_arches is not None: + if arch not in include_arches: continue - dep_settings = arch_caches.get(prof.sub_path) - if dep_settings is None: - dep_settings = portage.config( - config_profile_path=prof.abs_path, - config_incrementals=repoman_incrementals, - config_root=config_root, - local_config=False, - _unmatched_removal=options.unmatched_removal, - env=env) - dep_settings.categories = repoman_settings.categories - if options.without_mask: - dep_settings._mask_manager = \ - copy.deepcopy(dep_settings._mask_manager) - dep_settings._mask_manager._pmaskdict.clear() - arch_caches[prof.sub_path] = dep_settings - - xmatch_cache_key = (prof.sub_path, tuple(groups)) - xcache = arch_xmatch_caches.get(xmatch_cache_key) - if xcache is None: - portdb.melt() - portdb.freeze() - xcache = portdb.xcache - xcache.update(shared_xmatch_caches) - arch_xmatch_caches[xmatch_cache_key] = xcache - - trees[root]["porttree"].settings = dep_settings - portdb.settings = dep_settings - portdb.xcache = xcache - # for package.use.mask support inside dep_check - dep_settings.setcpv(pkg) - dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) - # just in case, prevent config.reset() from nuking these. - dep_settings.backup_changes("ACCEPT_KEYWORDS") - - if not baddepsyntax: - ismasked = not ebuild_archs or \ - pkg.cpv not in portdb.xmatch("match-visible", pkg.cp) - if ismasked: - if not have_pmasked: - have_pmasked = bool(dep_settings._getMaskAtom( - pkg.cpv, pkg.metadata)) - if options.ignore_masked: - continue - #we are testing deps for a masked package; give it some lee-way - suffix="masked" - matchmode = "minimum-all" - else: - suffix="" - matchmode = "minimum-visible" - - if not have_dev_keywords: - have_dev_keywords = \ - bool(dev_keywords.intersection(keywords)) - - if prof.status == "dev": - suffix=suffix+"indev" - - for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]: - - mykey=mytype+".bad"+suffix - myvalue = myaux[mytype] - if not myvalue: - continue - - success, atoms = portage.dep_check(myvalue, portdb, - dep_settings, use="all", mode=matchmode, - trees=trees) - - if success: - if atoms: - - # Don't bother with dependency.unknown for - # cases in which *DEPEND.bad is triggered. - for atom in atoms: - # dep_check returns all blockers and they - # aren't counted for *DEPEND.bad, so we - # ignore them here. - if not atom.blocker: - unknown_pkgs.discard( - (mytype, atom.unevaluated_atom)) - - if not prof.sub_path: - # old-style virtuals currently aren't - # resolvable with empty profile, since - # 'virtuals' mappings are unavailable - # (it would be expensive to search - # for PROVIDE in all ebuilds) - atoms = [atom for atom in atoms if not \ - (atom.cp.startswith('virtual/') and \ - not portdb.cp_list(atom.cp))] - - #we have some unsolvable deps - #remove ! deps, which always show up as unsatisfiable - atoms = [str(atom.unevaluated_atom) \ - for atom in atoms if not atom.blocker] - - #if we emptied out our list, continue: - if not atoms: - continue - stats[mykey]=stats[mykey]+1 - fails[mykey].append("%s: %s(%s) %s" % \ - (relative_path, keyword, - prof, repr(atoms))) - else: - stats[mykey]=stats[mykey]+1 - fails[mykey].append("%s: %s(%s) %s" % \ - (relative_path, keyword, + relevant_profiles.extend((keyword, groups, prof) + for prof in profiles[arch]) + + def sort_key(item): + return item[2].sub_path + + relevant_profiles.sort(key=sort_key) + + for keyword, groups, prof in relevant_profiles: + + if not (prof.status == "stable" or \ + (prof.status == "dev" and options.include_dev) or \ + (prof.status == "exp" and options.include_exp_profiles == 'y')): + continue + + dep_settings = arch_caches.get(prof.sub_path) + if dep_settings is None: + dep_settings = portage.config( + config_profile_path=prof.abs_path, + config_incrementals=repoman_incrementals, + config_root=config_root, + local_config=False, + _unmatched_removal=options.unmatched_removal, + env=env, repositories=repoman_settings.repositories) + dep_settings.categories = repoman_settings.categories + if options.without_mask: + dep_settings._mask_manager_obj = \ + copy.deepcopy(dep_settings._mask_manager) + dep_settings._mask_manager._pmaskdict.clear() + arch_caches[prof.sub_path] = dep_settings + + xmatch_cache_key = (prof.sub_path, tuple(groups)) + xcache = arch_xmatch_caches.get(xmatch_cache_key) + if xcache is None: + portdb.melt() + portdb.freeze() + xcache = portdb.xcache + xcache.update(shared_xmatch_caches) + arch_xmatch_caches[xmatch_cache_key] = xcache + + trees[root]["porttree"].settings = dep_settings + portdb.settings = dep_settings + portdb.xcache = xcache + + dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) + # just in case, prevent config.reset() from nuking these. + dep_settings.backup_changes("ACCEPT_KEYWORDS") + + # This attribute is used in dbapi._match_use() to apply + # use.stable.{mask,force} settings based on the stable + # status of the parent package. This is required in order + # for USE deps of unstable packages to be resolved correctly, + # since otherwise use.stable.{mask,force} settings of + # dependencies may conflict (see bug #456342). + dep_settings._parent_stable = dep_settings._isStable(pkg) + + # Handle package.use*.{force,mask) calculation, for use + # in dep_check. + dep_settings.useforce = dep_settings._use_manager.getUseForce( + pkg, stable=dep_settings._parent_stable) + dep_settings.usemask = dep_settings._use_manager.getUseMask( + pkg, stable=dep_settings._parent_stable) + + if not baddepsyntax: + ismasked = not ebuild_archs or \ + pkg.cpv not in portdb.xmatch("match-visible", pkg.cp) + if ismasked: + if not have_pmasked: + have_pmasked = bool(dep_settings._getMaskAtom( + pkg.cpv, pkg._metadata)) + if options.ignore_masked: + continue + # we are testing deps for a masked package; give it some lee-way + suffix = "masked" + matchmode = "minimum-all" + else: + suffix = "" + matchmode = "minimum-visible" + + if not have_dev_keywords: + have_dev_keywords = \ + bool(dev_keywords.intersection(keywords)) + + if prof.status == "dev": + suffix = suffix + "indev" + + for mytype in Package._dep_keys: + + mykey = "dependency.bad" + suffix + myvalue = myaux[mytype] + if not myvalue: + continue + + success, atoms = portage.dep_check(myvalue, portdb, + dep_settings, use="all", mode=matchmode, + trees=trees) + + if success: + if atoms: + + # Don't bother with dependency.unknown for + # cases in which *DEPEND.bad is triggered. + for atom in atoms: + # dep_check returns all blockers and they + # aren't counted for *DEPEND.bad, so we + # ignore them here. + if not atom.blocker: + unknown_pkgs.discard( + (mytype, atom.unevaluated_atom)) + + if not prof.sub_path: + # old-style virtuals currently aren't + # resolvable with empty profile, since + # 'virtuals' mappings are unavailable + # (it would be expensive to search + # for PROVIDE in all ebuilds) + atoms = [atom for atom in atoms if not \ + (atom.cp.startswith('virtual/') and \ + not portdb.cp_list(atom.cp))] + + # we have some unsolvable deps + # remove ! deps, which always show up as unsatisfiable + atoms = [str(atom.unevaluated_atom) \ + for atom in atoms if not atom.blocker] + + # if we emptied out our list, continue: + if not atoms: + continue + stats[mykey] += 1 + fails[mykey].append("%s: %s: %s(%s) %s" % \ + (relative_path, mytype, keyword, prof, repr(atoms))) + else: + stats[mykey] += 1 + fails[mykey].append("%s: %s: %s(%s) %s" % \ + (relative_path, mytype, keyword, + prof, repr(atoms))) if not baddepsyntax and unknown_pkgs: type_map = {} @@ -2178,11 +2390,11 @@ if options.if_modified == "y" and len(effective_scanlist) < 1: if options.mode == "manifest": sys.exit(dofail) -#dofail will be set to 1 if we have failed in at least one non-warning category -dofail=0 -#dowarn will be set to 1 if we tripped any warnings -dowarn=0 -#dofull will be set if we should print a "repoman full" informational message +# dofail will be set to 1 if we have failed in at least one non-warning category +dofail = 0 +# dowarn will be set to 1 if we tripped any warnings +dowarn = 0 +# dofull will be set if we should print a "repoman full" informational message dofull = options.mode != 'full' for x in qacats: @@ -2217,22 +2429,6 @@ del console_writer, f, style_file qa_output = qa_output.getvalue() qa_output = qa_output.splitlines(True) -def grouplist(mylist,seperator="/"): - """(list,seperator="/") -- Takes a list of elements; groups them into - same initial element categories. Returns a dict of {base:[sublist]} - From: ["blah/foo","spork/spatula","blah/weee/splat"] - To: {"blah":["foo","weee/splat"], "spork":["spatula"]}""" - mygroups={} - for x in mylist: - xs=x.split(seperator) - if xs[0]==".": - xs=xs[1:] - if xs[0] not in mygroups: - mygroups[xs[0]]=[seperator.join(xs[1:])] - else: - mygroups[xs[0]]+=[seperator.join(xs[1:])] - return mygroups - suggest_ignore_masked = False suggest_include_dev = False @@ -2281,65 +2477,65 @@ else: myunadded = [] if vcs == "cvs": try: - myvcstree=portage.cvstree.getentries("./",recursive=1) - myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./") + myvcstree = portage.cvstree.getentries("./", recursive=1) + myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./") except SystemExit as e: raise # TODO propagate this except: err("Error retrieving CVS tree; exiting.") if vcs == "svn": try: - with os.popen("svn status --no-ignore") as f: + with repoman_popen("svn status --no-ignore") as f: svnstatus = f.readlines() - myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ] + myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")] except SystemExit as e: raise # TODO propagate this except: err("Error retrieving SVN info; exiting.") if vcs == "git": # get list of files not under version control or missing - myf = os.popen("git ls-files --others") - myunadded = [ "./" + elem[:-1] for elem in myf ] + myf = repoman_popen("git ls-files --others") + myunadded = ["./" + elem[:-1] for elem in myf] myf.close() if vcs == "bzr": try: - with os.popen("bzr status -S .") as f: + with repoman_popen("bzr status -S .") as f: bzrstatus = f.readlines() - myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ] + myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"] except SystemExit as e: raise # TODO propagate this except: err("Error retrieving bzr info; exiting.") if vcs == "hg": - with os.popen("hg status --no-status --unknown .") as f: + with repoman_popen("hg status --no-status --unknown .") as f: myunadded = f.readlines() myunadded = ["./" + elem.rstrip() for elem in myunadded] - + # Mercurial doesn't handle manually deleted files as removed from # the repository, so the user need to remove them before commit, # using "hg remove [FILES]" - with os.popen("hg status --no-status --deleted .") as f: + with repoman_popen("hg status --no-status --deleted .") as f: mydeleted = f.readlines() mydeleted = ["./" + elem.rstrip() for elem in mydeleted] - myautoadd=[] + myautoadd = [] if myunadded: - for x in range(len(myunadded)-1,-1,-1): - xs=myunadded[x].split("/") - if xs[-1]=="files": + for x in range(len(myunadded)-1, -1, -1): + xs = myunadded[x].split("/") + if xs[-1] == "files": print("!!! files dir is not added! Please correct this.") sys.exit(-1) - elif xs[-1]=="Manifest": + elif xs[-1] == "Manifest": # It's a manifest... auto add - myautoadd+=[myunadded[x]] + myautoadd += [myunadded[x]] del myunadded[x] if myunadded: print(red("!!! The following files are in your local tree but are not added to the master")) print(red("!!! tree. Please remove them from the local tree or add them to the master tree.")) for x in myunadded: - print(" ",x) + print(" ", x) print() print() sys.exit(1) @@ -2348,7 +2544,7 @@ else: print(red("!!! The following files are removed manually from your local tree but are not")) print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\".")) for x in mydeleted: - print(" ",x) + print(" ", x) print() print() sys.exit(1) @@ -2357,60 +2553,59 @@ else: mycvstree = cvstree.getentries("./", recursive=1) mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./") - myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./") + myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./") bin_blob_pattern = re.compile("^-kb$") no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern, recursive=1, basedir="./")) - if vcs == "svn": - with os.popen("svn status") as f: + with repoman_popen("svn status") as f: svnstatus = f.readlines() - mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")] - mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")] - myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] + mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")] + mynew = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")] + myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] # Subversion expands keywords specified in svn:keywords properties. - with os.popen("svn propget -R svn:keywords") as f: + with repoman_popen("svn propget -R svn:keywords") as f: props = f.readlines() expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \ for prop in props if " - " in prop) elif vcs == "git": - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=M HEAD") as f: mychanged = f.readlines() mychanged = ["./" + elem[:-1] for elem in mychanged] - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=A HEAD") as f: mynew = f.readlines() mynew = ["./" + elem[:-1] for elem in mynew] - with os.popen("git diff-index --name-only " + with repoman_popen("git diff-index --name-only " "--relative --diff-filter=D HEAD") as f: myremoved = f.readlines() myremoved = ["./" + elem[:-1] for elem in myremoved] if vcs == "bzr": - with os.popen("bzr status -S .") as f: + with repoman_popen("bzr status -S .") as f: bzrstatus = f.readlines() - mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ] - mynew = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ] - myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ] - myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ] + mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"] + mynew = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")] + myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")] + myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")] # Bazaar expands nothing. if vcs == "hg": - with os.popen("hg status --no-status --modified .") as f: + with repoman_popen("hg status --no-status --modified .") as f: mychanged = f.readlines() mychanged = ["./" + elem.rstrip() for elem in mychanged] - with os.popen("hg status --no-status --added .") as f: + with repoman_popen("hg status --no-status --added .") as f: mynew = f.readlines() mynew = ["./" + elem.rstrip() for elem in mynew] - with os.popen("hg status --no-status --removed .") as f: + with repoman_popen("hg status --no-status --removed .") as f: myremoved = f.readlines() myremoved = ["./" + elem.rstrip() for elem in myremoved] @@ -2469,21 +2664,54 @@ else: commitmessage = commitmessage.rstrip() changelog_msg = commitmessage portage_version = getattr(portage, "VERSION", None) + gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "") + dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "") if portage_version is None: sys.stderr.write("Failed to insert portage version in message!\n") sys.stderr.flush() portage_version = "Unknown" - unameout = platform.system() + " " - if platform.system() in ["Darwin", "SunOS"]: - unameout += platform.processor() - else: - unameout += platform.machine() - commitmessage += "\n\n(Portage version: %s/%s/%s" % \ - (portage_version, vcs, unameout) + + report_options = [] if options.force: - commitmessage += ", RepoMan options: --force" - commitmessage += ")" + report_options.append("--force") + if options.ignore_arches: + report_options.append("--ignore-arches") + if include_arches is not None: + report_options.append("--include-arches=\"%s\"" % + " ".join(sorted(include_arches))) + if vcs == "git": + # Use new footer only for git (see bug #438364). + commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version + if report_options: + commit_footer += "\nRepoMan-Options: " + " ".join(report_options) + if sign_manifests: + commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, ) + if dco_sob: + commit_footer += "\nSigned-off-by: %s" % (dco_sob, ) + else: + unameout = platform.system() + " " + if platform.system() in ["Darwin", "SunOS"]: + unameout += platform.processor() + else: + unameout += platform.machine() + commit_footer = "\n\n" + if dco_sob: + commit_footer += "Signed-off-by: %s\n" % (dco_sob, ) + commit_footer += "(Portage version: %s/%s/%s" % \ + (portage_version, vcs, unameout) + if report_options: + commit_footer += ", RepoMan options: " + " ".join(report_options) + if sign_manifests: + commit_footer += ", signed Manifest commit with key %s" % \ + (gpg_key, ) + else: + commit_footer += ", unsigned Manifest commit" + commit_footer += ")" + + commitmessage += commit_footer + + broken_changelog_manifests = [] if options.echangelog in ('y', 'force'): logging.info("checking for unmodified ChangeLog files") committer_name = utilities.get_committer_name(env=repoman_settings) @@ -2535,6 +2763,13 @@ else: else: myupdates.append(changelog_path) + if options.ask and not options.pretend: + # regenerate Manifest for modified ChangeLog (bug #420735) + repoman_settings["O"] = checkdir + digestgen(mysettings=repoman_settings, myportdb=portdb) + else: + broken_changelog_manifests.append(x) + if myautoadd: print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") add_cmd = [vcs, "add"] @@ -2543,15 +2778,17 @@ else: portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd), noiselevel=-1) else: - if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000): - # Python 3.1 produces the following TypeError if raw bytes are - # passed to subprocess.call(): - # File "/usr/lib/python3.1/subprocess.py", line 646, in __init__ - # errread, errwrite) - # File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child - # raise child_exception - # TypeError: expected an object with the buffer interface - add_cmd = [_unicode_encode(arg) for arg in add_cmd] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(add_cmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(add_cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(add_cmd[0]) + add_cmd[0] = fullname + + add_cmd = [_unicode_encode(arg) for arg in add_cmd] retcode = subprocess.call(add_cmd) if retcode != os.EX_OK: logging.error( @@ -2596,7 +2833,7 @@ else: elif vcs == "svn": if myfile not in expansion: continue - + # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files. enabled_keywords = [] for k in expansion[myfile]: @@ -2606,7 +2843,8 @@ else: headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords) - myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile) + myout = repoman_getstatusoutput("egrep -q " + headerstring + " " + + portage._shell_quote(myfile)) if myout[0] == 0: myheaders.append(myfile) @@ -2653,7 +2891,7 @@ else: if options.pretend: print("(%s)" % (" ".join(commit_cmd),)) else: - retval = spawn(commit_cmd, env=os.environ) + retval = spawn(commit_cmd, env=commit_env) if retval != os.EX_OK: writemsg_level(("!!! Exiting on %s (shell) " + \ "error code: %s\n") % (vcs, retval), @@ -2694,14 +2932,38 @@ else: gpgvars[k] = v gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) if options.pretend: - print("("+gpgcmd+")") + print("(" + gpgcmd + ")") else: - rValue = os.system(gpgcmd) + # Encode unicode manually for bug #310789. + gpgcmd = portage.util.shlex_split(gpgcmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(gpgcmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(gpgcmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(gpgcmd[0]) + gpgcmd[0] = fullname + + gpgcmd = [_unicode_encode(arg, + encoding=_encodings['fs'], errors='strict') for arg in gpgcmd] + rValue = subprocess.call(gpgcmd) if rValue == os.EX_OK: - os.rename(filename+".asc", filename) + os.rename(filename + ".asc", filename) else: raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status") + def need_signature(filename): + try: + with open(_unicode_encode(filename, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + return b"BEGIN PGP SIGNED MESSAGE" not in f.readline() + except IOError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): + return False + raise + # When files are removed and re-added, the cvs server will put /Attic/ # inside the $Header path. This code detects the problem and corrects it # so that the Manifest will generate correctly. See bug #169500. @@ -2736,6 +2998,11 @@ else: repoman_settings["O"] = os.path.join(repodir, x) digestgen(mysettings=repoman_settings, myportdb=portdb) + elif broken_changelog_manifests: + for x in broken_changelog_manifests: + repoman_settings["O"] = os.path.join(repodir, x) + digestgen(mysettings=repoman_settings, myportdb=portdb) + signed = False if sign_manifests: signed = True @@ -2743,7 +3010,10 @@ else: for x in sorted(vcs_files_to_cps( chain(myupdates, myremoved, mymanifests))): repoman_settings["O"] = os.path.join(repodir, x) - gpgsign(os.path.join(repoman_settings["O"], "Manifest")) + manifest_path = os.path.join(repoman_settings["O"], "Manifest") + if not need_signature(manifest_path): + continue + gpgsign(manifest_path) except portage.exception.PortageException as e: portage.writemsg("!!! %s\n" % str(e)) portage.writemsg("!!! Disabled FEATURES='sign'\n") @@ -2771,7 +3041,6 @@ else: sys.exit(retval) if True: - myfiles = mymanifests[:] # If there are no header (SVN/CVS keywords) changes in # the files, this Manifest commit must include the @@ -2783,14 +3052,7 @@ else: fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "wb") - # strip the closing parenthesis - mymsg.write(_unicode_encode(commitmessage[:-1])) - if signed: - mymsg.write(_unicode_encode( - ", signed Manifest commit with key %s)" % \ - repoman_settings["PORTAGE_GPG_KEY"])) - else: - mymsg.write(b", unsigned Manifest commit)") + mymsg.write(_unicode_encode(commitmessage)) mymsg.close() commit_cmd = [] @@ -2813,9 +3075,8 @@ else: if options.pretend: print("(%s)" % (" ".join(commit_cmd),)) else: - retval = spawn(commit_cmd, env=os.environ) + retval = spawn(commit_cmd, env=commit_env) if retval != os.EX_OK: - if repo_config.sign_commit and vcs == 'git' and \ not git_supports_gpg_sign(): # Inform user that newer git is needed (bug #403323). @@ -2839,4 +3100,3 @@ else: print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything") print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n") sys.exit(0) - diff --git a/portage_with_autodep/bin/save-ebuild-env.sh b/portage_with_autodep/bin/save-ebuild-env.sh index 23e7aaa..7a31b89 100755 --- a/portage_with_autodep/bin/save-ebuild-env.sh +++ b/portage_with_autodep/bin/save-ebuild-env.sh @@ -1,8 +1,8 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# @FUNCTION: save_ebuild_env +# @FUNCTION: __save_ebuild_env # @DESCRIPTION: # echo the current environment to stdout, filtering out redundant info. # @@ -10,11 +10,12 @@ # be excluded from the output. These function are not needed for installation # or removal of the packages, and can therefore be safely excluded. # -save_ebuild_env() { +__save_ebuild_env() { ( if has --exclude-init-phases $* ; then unset S _E_DOCDESTTREE_ _E_EXEDESTTREE_ \ - PORTAGE_DOCOMPRESS PORTAGE_DOCOMPRESS_SKIP + PORTAGE_DOCOMPRESS_SIZE_LIMIT PORTAGE_DOCOMPRESS \ + PORTAGE_DOCOMPRESS_SKIP if [[ -n $PYTHONPATH && ${PYTHONPATH%%:*} -ef $PORTAGE_PYM_PATH ]] ; then if [[ $PYTHONPATH == *:* ]] ; then @@ -42,35 +43,51 @@ save_ebuild_env() { for x in pkg_setup pkg_nofetch src_unpack src_prepare src_configure \ src_compile src_test src_install pkg_preinst pkg_postinst \ pkg_prerm pkg_postrm ; do - unset -f default_$x _eapi{0,1,2,3,4}_$x + unset -f default_$x __eapi{0,1,2,3,4}_$x done unset x - unset -f assert assert_sigpipe_ok dump_trace die diefunc \ - quiet_mode vecho elog_base eqawarn elog \ - esyslog einfo einfon ewarn eerror ebegin _eend eend KV_major \ - KV_minor KV_micro KV_to_int get_KV unset_colors set_colors has \ - has_phase_defined_up_to \ - hasv hasq qa_source qa_call \ - addread addwrite adddeny addpredict _sb_append_var \ + unset -f assert __assert_sigpipe_ok \ + __dump_trace die \ + __quiet_mode __vecho __elog_base eqawarn elog \ + einfo einfon ewarn eerror ebegin __eend eend KV_major \ + KV_minor KV_micro KV_to_int get_KV __1 __1 has \ + __has_phase_defined_up_to \ + hasv hasq __qa_source __qa_call \ + addread addwrite adddeny addpredict __sb_append_var \ use usev useq has_version portageq \ best_version use_with use_enable register_die_hook \ - keepdir unpack strip_duplicate_slashes econf einstall \ - dyn_setup dyn_unpack dyn_clean into insinto exeinto docinto \ + unpack __strip_duplicate_slashes econf einstall \ + __dyn_setup __dyn_unpack __dyn_clean \ + into insinto exeinto docinto \ insopts diropts exeopts libopts docompress \ - abort_handler abort_prepare abort_configure abort_compile \ - abort_test abort_install dyn_prepare dyn_configure \ - dyn_compile dyn_test dyn_install \ - dyn_preinst dyn_pretend dyn_help debug-print debug-print-function \ - debug-print-section helpers_die inherit EXPORT_FUNCTIONS \ - nonfatal register_success_hook remove_path_entry \ - save_ebuild_env filter_readonly_variables preprocess_ebuild_env \ - set_unless_changed unset_unless_changed source_all_bashrcs \ - ebuild_main ebuild_phase ebuild_phase_with_hooks \ - _ebuild_arg_to_phase _ebuild_phase_funcs default \ - _hasg _hasgq _unpack_tar \ + __abort_handler __abort_prepare __abort_configure __abort_compile \ + __abort_test __abort_install __dyn_prepare __dyn_configure \ + __dyn_compile __dyn_test __dyn_install \ + __dyn_pretend __dyn_help \ + debug-print debug-print-function \ + debug-print-section __helpers_die inherit EXPORT_FUNCTIONS \ + nonfatal register_success_hook \ + __hasg __hasgq \ + __save_ebuild_env __set_colors __filter_readonly_variables \ + __preprocess_ebuild_env \ + __repo_key __source_all_bashrcs \ + __ebuild_main __ebuild_phase __ebuild_phase_with_hooks \ + __ebuild_arg_to_phase __ebuild_phase_funcs default \ + __unpack_tar __unset_colors \ ${QA_INTERCEPTORS} + ___eapi_has_usex && unset -f usex + ___eapi_has_master_repositories && unset -f master_repositories + ___eapi_has_repository_path && unset -f repository_path + ___eapi_has_available_eclasses && unset -f available_eclasses + ___eapi_has_eclass_path && unset -f eclass_path + ___eapi_has_license_path && unset -f license_path + ___eapi_has_package_manager_build_user && unset -f package_manager_build_user + ___eapi_has_package_manager_build_group && unset -f package_manager_build_group + + unset -f $(compgen -A function ___eapi_) + # portage config variables and variables set directly by portage unset ACCEPT_LICENSE BAD BRACKET BUILD_PREFIX COLS \ DISTCC_DIR DISTDIR DOC_SYMLINKS_DIR \ @@ -79,7 +96,11 @@ save_ebuild_env() { GOOD HILITE HOME \ LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ NOCOLOR NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \ - PORTAGE_BASHRCS_SOURCED PORTAGE_COMPRESS_EXCLUDE_SUFFIXES \ + PORTAGE_BASHRCS_SOURCED PORTAGE_COMPRESS \ + PORTAGE_COMPRESS_EXCLUDE_SUFFIXES \ + PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS \ + PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES \ + PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES \ PORTAGE_NONFATAL PORTAGE_QUIET \ PORTAGE_SANDBOX_DENY PORTAGE_SANDBOX_PREDICT \ PORTAGE_SANDBOX_READ PORTAGE_SANDBOX_WRITE PREROOTPATH \ diff --git a/portage_with_autodep/bin/xpak-helper.py b/portage_with_autodep/bin/xpak-helper.py index ef74920..d9d325d 100755 --- a/portage_with_autodep/bin/xpak-helper.py +++ b/portage_with_autodep/bin/xpak-helper.py @@ -1,11 +1,12 @@ #!/usr/bin/python -# Copyright 2009-2011 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import optparse import sys import portage +portage._internal_caller = True from portage import os +from portage.util._argparse import ArgumentParser def command_recompose(args): @@ -45,8 +46,8 @@ def main(argv): usage = "usage: %s COMMAND [args]" % \ os.path.basename(argv[0]) - parser = optparse.OptionParser(description=description, usage=usage) - options, args = parser.parse_args(argv[1:]) + parser = ArgumentParser(description=description, usage=usage) + options, args = parser.parse_known_args(argv[1:]) if not args: parser.error("missing command argument") diff --git a/portage_with_autodep/integration_with_portage.patch b/portage_with_autodep/integration_with_portage.patch deleted file mode 100644 index 794c156..0000000 --- a/portage_with_autodep/integration_with_portage.patch +++ /dev/null @@ -1,920 +0,0 @@ -diff -urN /usr/lib/portage/pym/_emerge/EbuildBuild.py ./pym/_emerge/EbuildBuild.py ---- /usr/lib/portage/pym/_emerge/EbuildBuild.py 2012-05-28 16:20:40.737712559 +0600 -+++ ./pym/_emerge/EbuildBuild.py 2012-06-01 21:37:22.799844102 +0600 -@@ -9,6 +9,8 @@ - from _emerge.EbuildMerge import EbuildMerge - from _emerge.EbuildFetchonly import EbuildFetchonly - from _emerge.EbuildBuildDir import EbuildBuildDir -+from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator -+from _emerge.EventsLogger import EventsLogger - from _emerge.MiscFunctionsProcess import MiscFunctionsProcess - from portage.util import writemsg - import portage -@@ -21,7 +23,7 @@ - class EbuildBuild(CompositeTask): - - __slots__ = ("args_set", "config_pool", "find_blockers", -- "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", -+ "ldpath_mtimes", "logger", "logserver", "opts", "pkg", "pkg_count", - "prefetcher", "settings", "world_atom") + \ - ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") - -@@ -251,8 +253,54 @@ - - build = EbuildExecuter(background=self.background, pkg=pkg, - scheduler=scheduler, settings=settings) -+ -+ build.addStartListener(self._build_start) -+ build.addExitListener(self._build_stop) -+ - self._start_task(build, self._build_exit) - -+ def _build_start(self,phase): -+ if "depcheck" in self.settings["FEATURES"] or \ -+ "depcheckstrict" in self.settings["FEATURES"]: -+ # Lets start a log listening server -+ temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) -+ -+ if "depcheckstrict" not in self.settings["FEATURES"]: -+ # use default filter_proc -+ self.logserver=EventsLogger(socket_dir=temp_path) -+ else: -+ portage.util.writemsg("Getting list of allowed files..." + \ -+ "This may take some time\n") -+ filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) -+ filter_proc=filter_gen.get_filter_proc() -+ self.logserver=EventsLogger(socket_dir=temp_path, -+ filter_proc=filter_proc) -+ -+ self.logserver.start() -+ -+ # Copy socket path to LOG_SOCKET environment variable -+ env=self.settings.configdict["pkg"] -+ env['LOG_SOCKET'] = self.logserver.socket_name -+ -+ #import pdb; pdb.set_trace() -+ -+ def _build_stop(self,phase): -+ if "depcheck" in self.settings["FEATURES"] or \ -+ "depcheckstrict" in self.settings["FEATURES"]: -+ # Delete LOG_SOCKET from environment -+ env=self.settings.configdict["pkg"] -+ if 'LOG_SOCKET' in env: -+ del env['LOG_SOCKET'] -+ -+ events=self.logserver.stop() -+ self.logserver=None -+ analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) -+ analyser.display() # show the analyse -+ -+ #import pdb; pdb.set_trace() -+ -+ -+ - def _fetch_failed(self): - # We only call the pkg_nofetch phase if either RESTRICT=fetch - # is set or the package has explicitly overridden the default -diff -urN /usr/lib/portage/pym/_emerge/EbuildPhase.py ./pym/_emerge/EbuildPhase.py ---- /usr/lib/portage/pym/_emerge/EbuildPhase.py 2012-05-28 16:20:40.738712559 +0600 -+++ ./pym/_emerge/EbuildPhase.py 2012-06-01 21:38:11.935842036 +0600 -@@ -34,7 +34,8 @@ - - # FEATURES displayed prior to setup phase - _features_display = ( -- "ccache", "compressdebug", "distcc", "distcc-pump", "fakeroot", -+ "ccache", "compressdebug", "depcheck", "depcheckstrict", -+ "distcc", "distcc-pump", "fakeroot", - "installsources", "keeptemp", "keepwork", "nostrip", - "preserve-libs", "sandbox", "selinux", "sesandbox", - "splitdebug", "suidctl", "test", "userpriv", -diff -urN /usr/lib/portage/pym/_emerge/EbuildPhase.py.rej ./pym/_emerge/EbuildPhase.py.rej ---- /usr/lib/portage/pym/_emerge/EbuildPhase.py.rej 1970-01-01 05:00:00.000000000 +0500 -+++ ./pym/_emerge/EbuildPhase.py.rej 2012-06-01 21:37:22.800844102 +0600 -@@ -0,0 +1,12 @@ -+--- pym/_emerge/EbuildPhase.py -++++ pym/_emerge/EbuildPhase.py -+@@ -33,7 +33,8 @@ -+ ("_ebuild_lock",) -+ -+ # FEATURES displayed prior to setup phase -+- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", -++ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", -++ "distcc-pump", "fakeroot", -+ "installsources", "keeptemp", "keepwork", "nostrip", -+ "preserve-libs", "sandbox", "selinux", "sesandbox", -+ "splitdebug", "suidctl", "test", "userpriv", -diff -urN /usr/lib/portage/pym/_emerge/EventsAnalyser.py ./pym/_emerge/EventsAnalyser.py ---- /usr/lib/portage/pym/_emerge/EventsAnalyser.py 1970-01-01 05:00:00.000000000 +0500 -+++ ./pym/_emerge/EventsAnalyser.py 2012-06-01 21:37:22.802844102 +0600 -@@ -0,0 +1,511 @@ -+# Distributed under the terms of the GNU General Public License v2 -+ -+import portage -+from portage.dbapi._expand_new_virt import expand_new_virt -+from portage import os -+ -+import subprocess -+import re -+ -+class PortageUtils: -+ """ class for accessing the portage api """ -+ def __init__(self, settings): -+ """ test """ -+ self.settings=settings -+ self.vartree=portage.vartree(settings=settings) -+ self.vardbapi=portage.vardbapi(settings=settings, vartree=self.vartree) -+ self.portdbapi=portage.portdbapi(mysettings=settings) -+ self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")] -+ self.use=self.settings["USE"] -+ -+ def get_best_visible_pkg(self,pkg): -+ """ -+ Gets best candidate on installing. Returns empty string if no found -+ -+ :param pkg: package name -+ -+ """ -+ try: -+ return self.portdbapi.xmatch("bestmatch-visible", pkg) -+ except: -+ return '' -+ -+ # non-recursive dependency getter -+ def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): -+ """ -+ Gets current dependencies of a package. Looks in portage db -+ -+ :param pkg: name of package -+ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or -+ ["RDEPEND", "DEPEND"] -+ :returns: **set** of packages names -+ """ -+ ret=set() -+ -+ pkg = self.get_best_visible_pkg(pkg) -+ if not pkg: -+ return ret -+ -+ # we found the best visible match in common tree -+ -+ -+ metadata = dict(zip(self.metadata_keys, -+ self.portdbapi.aux_get(pkg, self.metadata_keys))) -+ dep_str = " ".join(metadata[k] for k in dep_type) -+ -+ # the IUSE default are very important for us -+ iuse_defaults=[ -+ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] -+ -+ use=self.use.split() -+ -+ for u in iuse_defaults: -+ if u not in use: -+ use.append(u) -+ -+ success, atoms = portage.dep_check(dep_str, None, self.settings, -+ myuse=use, myroot=self.settings["ROOT"], -+ trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) -+ if not success: -+ return ret -+ -+ for atom in atoms: -+ atomname = self.vartree.dep_bestmatch(atom) -+ -+ if not atomname: -+ continue -+ -+ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): -+ for pkg in self.vartree.dep_match(unvirt_pkg): -+ ret.add(pkg) -+ -+ return ret -+ -+ # recursive dependency getter -+ def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): -+ """ -+ Gets current dependencies of a package on any depth -+ All dependencies **must** be installed -+ -+ :param pkg: name of package -+ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or -+ ["RDEPEND", "DEPEND"] -+ :returns: **set** of packages names -+ """ -+ ret=set() -+ -+ -+ # get porttree dependencies on the first package -+ -+ pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) -+ if not pkg: -+ return ret -+ -+ known_packages=set() -+ unknown_packages=self.get_dep(pkg,dep_type) -+ ret=ret.union(unknown_packages) -+ -+ while unknown_packages: -+ p=unknown_packages.pop() -+ if p in known_packages: -+ continue -+ known_packages.add(p) -+ -+ metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) -+ -+ dep_str = " ".join(metadata[k] for k in dep_type) -+ -+ # the IUSE default are very important for us -+ iuse_defaults=[ -+ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] -+ -+ use=self.use.split() -+ -+ for u in iuse_defaults: -+ if u not in use: -+ use.append(u) -+ -+ success, atoms = portage.dep_check(dep_str, None, self.settings, -+ myuse=use, myroot=self.settings["ROOT"], -+ trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) -+ -+ if not success: -+ continue -+ -+ for atom in atoms: -+ atomname = self.vartree.dep_bestmatch(atom) -+ if not atomname: -+ continue -+ -+ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): -+ for pkg in self.vartree.dep_match(unvirt_pkg): -+ ret.add(pkg) -+ unknown_packages.add(pkg) -+ return ret -+ -+ def get_deps_for_package_building(self, pkg): -+ """ -+ returns buildtime dependencies of current package and -+ all runtime dependencies of that buildtime dependencies -+ """ -+ buildtime_deps=self.get_dep(pkg, ["DEPEND"]) -+ runtime_deps=set() -+ for dep in buildtime_deps: -+ runtime_deps=runtime_deps.union(self.get_deps(dep,["RDEPEND"])) -+ -+ ret=buildtime_deps.union(runtime_deps) -+ return ret -+ -+ def get_system_packages_list(self): -+ """ -+ returns all packages from system set. They are always implicit dependencies -+ -+ :returns: **list** of package names -+ """ -+ ret=[] -+ for atom in self.settings.packages: -+ for pre_pkg in self.vartree.dep_match(atom): -+ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+pre_pkg): -+ for pkg in self.vartree.dep_match(unvirt_pkg): -+ ret.append(pkg) -+ return ret -+ -+ -+class GentoolkitUtils: -+ """ -+ Interface with qfile and qlist utils. They are much faster than -+ internals. -+ """ -+ -+ def getpackagesbyfiles(files): -+ """ -+ :param files: list of filenames -+ :returns: **dictionary** file->package, if file doesn't belong to any -+ package it not returned as key of this dictionary -+ """ -+ ret={} -+ listtocheck=[] -+ for f in files: -+ if os.path.isdir(f): -+ ret[f]="directory" -+ else: -+ listtocheck.append(f) -+ -+ try: -+ proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], -+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, -+ bufsize=4096) -+ -+ out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) -+ -+ lines=out.decode("utf8").split("\n") -+ #print lines -+ line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") -+ for line in lines: -+ if len(line)==0: -+ continue -+ match=line_re.match(line) -+ if match: -+ ret[match.group(2)]=match.group(1) -+ else: -+ portage.util.writemsg("Util qfile returned unparsable string: %s\n" % line) -+ -+ except OSError as e: -+ portage.util.writemsg("Error while launching qfile: %s\n" % e) -+ -+ -+ return ret -+ -+ def getfilesbypackages(packagenames): -+ """ -+ -+ :param packagename: name of package -+ :returns: **list** of files in package with name *packagename* -+ """ -+ ret=[] -+ try: -+ proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, -+ stdout=subprocess.PIPE,stderr=subprocess.PIPE, -+ bufsize=4096) -+ -+ out,err=proc.communicate() -+ -+ ret=out.decode("utf8").split("\n") -+ if ret==['']: -+ ret=[] -+ except OSError as e: -+ portage.util.writemsg("Error while launching qfile: %s\n" % e) -+ -+ return ret -+ -+ def get_all_packages_files(): -+ """ -+ Memory-hungry operation -+ -+ :returns: **set** of all files that belongs to package -+ """ -+ ret=[] -+ try: -+ proc=subprocess.Popen(['qlist']+['--all',"--obj"], -+ stdout=subprocess.PIPE,stderr=subprocess.PIPE, -+ bufsize=4096) -+ -+ out,err=proc.communicate() -+ -+ ret=out.decode("utf8").split("\n") -+ except OSError as e: -+ portage.util.writemsg("Error while launching qfile: %s\n" % e) -+ -+ return set(ret) -+ -+class FilterProcGenerator: -+ def __init__(self, pkgname, settings): -+ portageutils=PortageUtils(settings=settings) -+ -+ deps_all=portageutils.get_deps_for_package_building(pkgname) -+ deps_portage=portageutils.get_dep('portage',["RDEPEND"]) -+ -+ system_packages=portageutils.get_system_packages_list() -+ -+ allfiles=GentoolkitUtils.get_all_packages_files() -+ portage.util.writemsg("All files list recieved, waiting for " \ -+ "a list of allowed files\n") -+ -+ -+ allowedpkgs=system_packages+list(deps_portage)+list(deps_all) -+ -+ allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) -+ #for pkg in allowedpkgs: -+ # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) -+ -+ #import pdb; pdb.set_trace() -+ -+ # manually add all python interpreters to this list -+ allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) -+ allowedfiles=set(allowedfiles) -+ -+ deniedfiles=allfiles-allowedfiles -+ -+ def filter_proc(eventname,filename,stage): -+ if filename in deniedfiles: -+ return False -+ return True -+ -+ self.filter_proc=filter_proc -+ def get_filter_proc(self): -+ return self.filter_proc -+ -+class EventsAnalyser: -+ def __init__(self, pkgname, events, settings): -+ self.pkgname=pkgname -+ self.events=events -+ self.settings=settings -+ self.portageutils=PortageUtils(settings=settings) -+ -+ self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) -+ self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) -+ self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) -+ -+ self.system_packages=self.portageutils.get_system_packages_list() -+ # All analyse work is here -+ -+ # get unique filenames -+ filenames=set() -+ for stage in events: -+ succ_events=set(events[stage][0]) -+ fail_events=set(events[stage][1]) -+ filenames=filenames.union(succ_events) -+ filenames=filenames.union(fail_events) -+ filenames=list(filenames) -+ -+ file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) -+ # This part is completly unreadable. -+ # It converting one complex struct(returned by getfsevents) to another complex -+ # struct which good for generating output. -+ # -+ # Old struct is also used during output -+ -+ packagesinfo={} -+ -+ for stage in sorted(events): -+ succ_events=events[stage][0] -+ fail_events=events[stage][1] -+ -+ for filename in succ_events: -+ if filename in file_to_package: -+ package=file_to_package[filename] -+ else: -+ package="unknown" -+ -+ if not package in packagesinfo: -+ packagesinfo[package]={} -+ stageinfo=packagesinfo[package] -+ if not stage in stageinfo: -+ stageinfo[stage]={} -+ -+ filesinfo=stageinfo[stage] -+ if not filename in filesinfo: -+ filesinfo[filename]={"found":[],"notfound":[]} -+ filesinfo[filename]["found"]=succ_events[filename] -+ -+ for filename in fail_events: -+ if filename in file_to_package: -+ package=file_to_package[filename] -+ else: -+ package="unknown" -+ if not package in packagesinfo: -+ packagesinfo[package]={} -+ stageinfo=packagesinfo[package] -+ if not stage in stageinfo: -+ stageinfo[stage]={} -+ -+ filesinfo=stageinfo[stage] -+ if not filename in filesinfo: -+ filesinfo[filename]={"found":[],"notfound":[]} -+ filesinfo[filename]["notfound"]=fail_events[filename] -+ self.packagesinfo=packagesinfo -+ -+ def display(self): -+ portage.util.writemsg( -+ portage.output.colorize( -+ "WARN", "\nFile access report for %s:\n" % self.pkgname)) -+ -+ stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, -+ "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} -+ packagesinfo=self.packagesinfo -+ # print information grouped by package -+ for package in sorted(packagesinfo): -+ # not showing special directory package -+ if package=="directory": -+ continue -+ -+ if package=="unknown": -+ continue -+ -+ -+ is_pkg_in_dep=package in self.deps_all -+ is_pkg_in_portage_dep=package in self.deps_portage -+ is_pkg_in_system=package in self.system_packages -+ is_pkg_python="dev-lang/python" in package -+ -+ stages=[] -+ for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get): -+ if stage!="unknown": -+ stages.append(stage) -+ -+ if len(stages)==0: -+ continue -+ -+ filenames={} -+ for stage in stages: -+ for filename in packagesinfo[package][stage]: -+ if len(packagesinfo[package][stage][filename]["found"])!=0: -+ was_readed,was_writed=packagesinfo[package][stage][filename]["found"] -+ if not filename in filenames: -+ filenames[filename]=['ok',was_readed,was_writed] -+ else: -+ status, old_was_readed, old_was_writed=filenames[filename] -+ filenames[filename]=[ -+ 'ok',old_was_readed | was_readed, old_was_writed | was_writed -+ ] -+ if len(packagesinfo[package][stage][filename]["notfound"])!=0: -+ was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] -+ if not filename in filenames: -+ filenames[filename]=['err',was_notfound,was_blocked] -+ else: -+ status, old_was_notfound, old_was_blocked=filenames[filename] -+ filenames[filename]=[ -+ 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked -+ ] -+ -+ -+ if is_pkg_in_dep: -+ portage.util.writemsg("[OK]") -+ elif is_pkg_in_system: -+ portage.util.writemsg("[SYSTEM]") -+ elif is_pkg_in_portage_dep: -+ portage.util.writemsg("[PORTAGE DEP]") -+ elif is_pkg_python: -+ portage.util.writemsg("[INTERPRETER]") -+ elif not self.is_package_useful(package,stages,filenames.keys()): -+ portage.util.writemsg("[LIKELY OK]") -+ else: -+ portage.util.writemsg(portage.output.colorize("BAD", "[NOT IN DEPS]")) -+ # show information about accessed files -+ -+ portage.util.writemsg(" %-40s: %s\n" % (package,stages)) -+ -+ # this is here for readability -+ action={ -+ ('ok',False,False):"accessed", -+ ('ok',True,False):"readed", -+ ('ok',False,True):"writed", -+ ('ok',True,True):"readed and writed", -+ ('err',False,False):"other error", -+ ('err',True,False):"not found", -+ ('err',False,True):"blocked", -+ ('err',True,True):"not found and blocked" -+ } -+ -+ filescounter=0 -+ -+ for filename in filenames: -+ event_info=tuple(filenames[filename]) -+ portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) -+ filescounter+=1 -+ if filescounter>10: -+ portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) -+ break -+ # ... and one more check. Making sure that direct build time -+ # dependencies were accessed -+ #import pdb; pdb.set_trace() -+ not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) -+ if not_accessed_deps: -+ portage.util.writemsg(portage.output.colorize("WARN", "!!! ")) -+ portage.util.writemsg("Warning! Some build time dependencies " + \ -+ "of packages were not accessed: " + \ -+ " ".join(not_accessed_deps) + "\n") -+ -+ def is_package_useful(self,pkg,stages,files): -+ """ some basic heuristics here to cut part of packages """ -+ -+ excluded_paths=set( -+ ['/etc/sandbox.d/'] -+ ) -+ -+ excluded_packages=set( -+ # autodep shows these two packages every time -+ ['net-zope/zope-fixers', 'net-zope/zope-interface'] -+ ) -+ -+ -+ def is_pkg_excluded(p): -+ for pkg in excluded_packages: -+ if p.startswith(pkg): # if package is excluded -+ return True -+ return False -+ -+ -+ def is_file_excluded(f): -+ for path in excluded_paths: -+ if f.startswith(path): # if path is excluded -+ return True -+ return False -+ -+ -+ if is_pkg_excluded(pkg): -+ return False -+ -+ for f in files: -+ if is_file_excluded(f): -+ continue -+ -+ # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 -+ if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): -+ break -+ else: -+ return False # we get here if cycle ends not with break -+ -+ return True -+ -+ -\ No newline at end of file -diff -urN /usr/lib/portage/pym/_emerge/EventsLogger.py ./pym/_emerge/EventsLogger.py ---- /usr/lib/portage/pym/_emerge/EventsLogger.py 1970-01-01 05:00:00.000000000 +0500 -+++ ./pym/_emerge/EventsLogger.py 2012-06-01 21:37:22.803844102 +0600 -@@ -0,0 +1,180 @@ -+# Distributed under the terms of the GNU General Public License v2 -+ -+import io -+import sys -+import stat -+import socket -+import select -+import tempfile -+ -+import threading -+ -+from portage import os -+ -+class EventsLogger(threading.Thread): -+ def default_filter(eventname, filename, stage): -+ return True -+ -+ def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): -+ threading.Thread.__init__(self) # init the Thread -+ -+ self.alive=False -+ -+ self.main_thread=threading.currentThread() -+ -+ self.socket_dir=socket_dir -+ self.filter_proc=filter_proc -+ -+ self.socket_name=None -+ self.socket_logger=None -+ -+ self.events={} -+ -+ try: -+ socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, -+ prefix="log_socket_") -+ -+ socket_name = os.path.join(socket_dir_name, 'socket') -+ -+ except OSError as e: -+ return -+ -+ self.socket_name=socket_name -+ -+ #print(self.socket_name) -+ -+ try: -+ socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) -+ socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -+ -+ socket_logger.bind(self.socket_name) -+ socket_logger.listen(64) -+ -+ except socket.error as e: -+ return -+ -+ self.socket_logger=socket_logger -+ -+ try: -+ # Allow connecting to socket for anyone -+ os.chmod(socket_dir_name, -+ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| -+ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) -+ os.chmod(socket_name, -+ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| -+ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) -+ except OSError as e: -+ return -+ -+ def run(self): -+ """ Starts the log server """ -+ -+ self.alive=True -+ self.listen_thread=threading.currentThread() -+ clients={} -+ -+ epoll=select.epoll() -+ epoll.register(self.socket_logger.fileno(), select.EPOLLIN) -+ -+ while self.alive: -+ try: -+ sock_events = epoll.poll(3) -+ -+ for fileno, sock_event in sock_events: -+ if fileno == self.socket_logger.fileno(): -+ ret = self.socket_logger.accept() -+ if ret is None: -+ pass -+ else: -+ (client,addr)=ret -+ epoll.register(client.fileno(), select.EPOLLIN) -+ clients[client.fileno()]=client -+ elif sock_event & select.EPOLLIN: -+ s=clients[fileno] -+ record=s.recv(8192) -+ -+ if not record: # if connection was closed -+ epoll.unregister(fileno) -+ clients[fileno].close() -+ del clients[fileno] -+ continue -+ -+ #import pdb; pdb.set_trace() -+ try: -+ message=record.decode("utf8").split("\0") -+ except UnicodeDecodeError: -+ print("Bad message %s" % record) -+ continue -+ -+ # continue -+ -+ #print(message) -+ -+ try: -+ if message[4]=="ASKING": -+ if self.filter_proc(message[1],message[2],message[3]): -+ s.sendall(b"ALLOW\0") -+ else: -+ # TODO: log through portage infrastructure -+ #print("Blocking an access to %s" % message[2]) -+ s.sendall(b"DENY\0") -+ else: -+ eventname,filename,stage,result=message[1:5] -+ -+ if not stage in self.events: -+ self.events[stage]=[{},{}] -+ -+ hashofsucesses=self.events[stage][0] -+ hashoffailures=self.events[stage][1] -+ -+ if result=="DENIED": -+ print("Blocking an access to %s" % filename) -+ -+ if result=="OK": -+ if not filename in hashofsucesses: -+ hashofsucesses[filename]=[False,False] -+ -+ readed_or_writed=hashofsucesses[filename] -+ -+ if eventname=="read": -+ readed_or_writed[0]=True -+ elif eventname=="write": -+ readed_or_writed[1]=True -+ -+ elif result[0:3]=="ERR" or result=="DENIED": -+ if not filename in hashoffailures: -+ hashoffailures[filename]=[False,False] -+ notfound_or_blocked=hashoffailures[filename] -+ -+ if result=="ERR/2": -+ notfound_or_blocked[0]=True -+ elif result=="DENIED": -+ notfound_or_blocked[1]=True -+ -+ else: -+ print("Error in logger module<->analyser protocol") -+ -+ except IndexError: -+ print("IndexError while parsing %s" % record) -+ except IOError as e: -+ if e.errno!=4: # handling "Interrupted system call" errors -+ raise -+ -+ # if main thread doesnt exists then exit -+ if not self.main_thread.is_alive(): -+ break -+ epoll.unregister(self.socket_logger.fileno()) -+ epoll.close() -+ self.socket_logger.close() -+ -+ def stop(self): -+ """ Stops the log server. Returns all events """ -+ -+ self.alive=False -+ -+ # Block the main thread until listener exists -+ self.listen_thread.join() -+ -+ # We assume portage clears tmp folder, so no deleting a socket file -+ # We assume that no new socket data will arrive after this moment -+ return self.events -diff -urN /usr/lib/portage/pym/portage/const.py ./pym/portage/const.py ---- /usr/lib/portage/pym/portage/const.py 2012-05-28 16:20:40.766712558 +0600 -+++ ./pym/portage/const.py 2012-06-01 21:39:51.363837853 +0600 -@@ -67,6 +67,8 @@ - BASH_BINARY = "/bin/bash" - MOVE_BINARY = "/bin/mv" - PRELINK_BINARY = "/usr/sbin/prelink" -+AUTODEP_LIBRARY = "/usr/lib/file_hook.so" -+ - - INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" - REPO_NAME_FILE = "repo_name" -@@ -89,7 +91,7 @@ - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", - "ccache", "chflags", "clean-logs", - "collision-protect", "compress-build-logs", "compressdebug", -- "config-protect-if-modified", -+ "config-protect-if-modified", "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "force-mirror", "force-prefix", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", -diff -urN /usr/lib/portage/pym/portage/const.py.rej ./pym/portage/const.py.rej ---- /usr/lib/portage/pym/portage/const.py.rej 1970-01-01 05:00:00.000000000 +0500 -+++ ./pym/portage/const.py.rej 2012-06-01 21:37:22.803844102 +0600 -@@ -0,0 +1,12 @@ -+--- pym/portage/const.py -++++ pym/portage/const.py -+@@ -90,7 +92,8 @@ -+ SUPPORTED_FEATURES = frozenset([ -+ "allow-missing-manifests", -+ "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", -+- "ccache", "chflags", "collision-protect", "compress-build-logs", -++ "ccache", "chflags", "collision-protect", "compress-build-logs", -++ "depcheck", "depcheckstrict", -+ "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", -+ "fail-clean", "fixpackages", "force-mirror", "getbinpkg", -+ "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", -diff -urN /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py ./pym/portage/package/ebuild/_config/special_env_vars.py ---- /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py 2012-05-28 16:20:40.870712551 +0600 -+++ ./pym/portage/package/ebuild/_config/special_env_vars.py 2012-06-01 21:37:22.804844102 +0600 -@@ -101,8 +101,8 @@ - # other variables inherited from the calling environment - environ_whitelist += [ - "CVS_RSH", "ECHANGELOG_USER", -- "GPG_AGENT_INFO", -- "SSH_AGENT_PID", "SSH_AUTH_SOCK", -+ "GPG_AGENT_INFO", "LOG_SOCKET", -+ "SSH_AGENT_PID", "SSH_AUTH_SOCK" - "STY", "WINDOW", "XAUTHORITY", - ] - -diff -urN /usr/lib/portage/pym/portage/package/ebuild/doebuild.py ./pym/portage/package/ebuild/doebuild.py ---- /usr/lib/portage/pym/portage/package/ebuild/doebuild.py 2012-05-28 16:20:40.860712554 +0600 -+++ ./pym/portage/package/ebuild/doebuild.py 2012-06-01 21:37:22.805844102 +0600 -@@ -1222,6 +1222,9 @@ - nosandbox = ("sandbox" not in features and \ - "usersandbox" not in features) - -+ if "depcheck" in features or "depcheckstrict" in features: -+ nosandbox = True -+ - if not portage.process.sandbox_capable: - nosandbox = True - -@@ -1401,7 +1404,10 @@ - keywords["opt_name"] = "[%s/%s]" % \ - (mysettings.get("CATEGORY",""), mysettings.get("PF","")) - -- if free or "SANDBOX_ACTIVE" in os.environ: -+ if "depcheck" in features or "depcheckstrict" in features: -+ keywords["opt_name"] += " bash" -+ spawn_func = portage.process.spawn_autodep -+ elif free or "SANDBOX_ACTIVE" in os.environ: - keywords["opt_name"] += " bash" - spawn_func = portage.process.spawn_bash - elif fakeroot: -diff -urN /usr/lib/portage/pym/portage/process.py ./pym/portage/process.py ---- /usr/lib/portage/pym/portage/process.py 2012-05-28 16:20:40.768712558 +0600 -+++ ./pym/portage/process.py 2012-06-01 21:37:22.806844102 +0600 -@@ -18,7 +18,7 @@ - 'portage.util:dump_traceback', - ) - --from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY -+from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY - from portage.exception import CommandNotFound - - try: -@@ -53,6 +53,9 @@ - sandbox_capable = (os.path.isfile(SANDBOX_BINARY) and - os.access(SANDBOX_BINARY, os.X_OK)) - -+autodep_capable = (os.path.isfile(AUTODEP_LIBRARY) and -+ os.access(AUTODEP_LIBRARY, os.X_OK)) -+ - fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and - os.access(FAKEROOT_BINARY, os.X_OK)) - -@@ -80,6 +83,16 @@ - args.append(mycommand) - return spawn(args, opt_name=opt_name, **keywords) - -+def spawn_autodep(mycommand, opt_name=None, **keywords): -+ if not autodep_capable: -+ return spawn_bash(mycommand, opt_name=opt_name, **keywords) -+ if "env" not in keywords or "LOG_SOCKET" not in keywords["env"]: -+ return spawn_bash(mycommand, opt_name=opt_name, **keywords) -+ -+ # Core part: tell the loader to preload logging library -+ keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY -+ return spawn_bash(mycommand, opt_name=opt_name, **keywords) -+ - def spawn_sandbox(mycommand, opt_name=None, **keywords): - if not sandbox_capable: - return spawn_bash(mycommand, opt_name=opt_name, **keywords) diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py index 94f26ef..1fcd043 100644 --- a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py @@ -1,11 +1,12 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import copy from portage.util.SlotObject import SlotObject class AbstractDepPriority(SlotObject): - __slots__ = ("buildtime", "runtime", "runtime_post") + __slots__ = ("buildtime", "buildtime_slot_op", + "runtime", "runtime_post", "runtime_slot_op") def __lt__(self, other): return self.__int__() < other diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo Binary files differindex b6a9871..0108460 100644 --- a/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py index c7b8f83..31127f4 100644 --- a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py @@ -1,8 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import stat +import subprocess import textwrap from _emerge.SpawnProcess import SpawnProcess from _emerge.EbuildBuildDir import EbuildBuildDir @@ -20,8 +22,10 @@ class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') + _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) _phases_interactive_whitelist = ('config',) + _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It @@ -52,13 +56,48 @@ class AbstractEbuildProcess(SpawnProcess): if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " - "since PORTAGE_BUILDIR does not exist: '%s'") % \ + "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return + # Check if the cgroup hierarchy is in place. If it's not, mount it. + if (os.geteuid() == 0 and platform.system() == 'Linux' + and 'cgroup' in self.settings.features + and self.phase not in self._phases_without_cgroup): + cgroup_root = '/sys/fs/cgroup' + cgroup_portage = os.path.join(cgroup_root, 'portage') + cgroup_path = os.path.join(cgroup_portage, + '%s:%s' % (self.settings["CATEGORY"], + self.settings["PF"])) + try: + # cgroup tmpfs + if not os.path.ismount(cgroup_root): + # we expect /sys/fs to be there already + if not os.path.isdir(cgroup_root): + os.mkdir(cgroup_root, 0o755) + subprocess.check_call(['mount', '-t', 'tmpfs', + '-o', 'rw,nosuid,nodev,noexec,mode=0755', + 'tmpfs', cgroup_root]) + + # portage subsystem + if not os.path.ismount(cgroup_portage): + if not os.path.isdir(cgroup_portage): + os.mkdir(cgroup_portage, 0o755) + subprocess.check_call(['mount', '-t', 'cgroup', + '-o', 'rw,nosuid,nodev,noexec,none,name=portage', + 'tmpfs', cgroup_portage]) + + # the ebuild cgroup + if not os.path.isdir(cgroup_path): + os.mkdir(cgroup_path) + except (subprocess.CalledProcessError, OSError): + pass + else: + self.cgroup = cgroup_path + if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. @@ -67,7 +106,7 @@ class AbstractEbuildProcess(SpawnProcess): if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: - if 'PORTAGE_BUILDIR_LOCKED' not in self.settings: + if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() @@ -143,9 +182,14 @@ class AbstractEbuildProcess(SpawnProcess): self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { - 'best_version' : query_command, - 'exit' : self._exit_command, - 'has_version' : query_command, + 'available_eclasses' : query_command, + 'best_version' : query_command, + 'eclass_path' : query_command, + 'exit' : self._exit_command, + 'has_version' : query_command, + 'license_path' : query_command, + 'master_repositories' : query_command, + 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo Binary files differindex b55f9c2..35828fe 100644 --- a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.py b/portage_with_autodep/pym/_emerge/AbstractPollTask.py index 2c84709..3f6dd6c 100644 --- a/portage_with_autodep/pym/_emerge/AbstractPollTask.py +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.py @@ -151,4 +151,4 @@ class AbstractPollTask(AsynchronousTask): while self._registered and not timeout_cb.timed_out: self.scheduler.iteration() finally: - self.scheduler.unregister(timeout_cb.timeout_id) + self.scheduler.source_remove(timeout_cb.timeout_id) diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo Binary files differindex 06ef6b9..c6208bd 100644 --- a/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.py b/portage_with_autodep/pym/_emerge/AsynchronousLock.py index 587aa46..c0b9b26 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousLock.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import dummy_threading @@ -49,7 +49,7 @@ class AsynchronousLock(AsynchronousTask): pass else: self.returncode = os.EX_OK - self.wait() + self._async_wait() return if self._force_process or \ @@ -105,44 +105,27 @@ class _LockThread(AbstractPollTask): """ __slots__ = ('path',) + \ - ('_files', '_force_dummy', '_lock_obj', - '_thread', '_reg_id',) + ('_force_dummy', '_lock_obj', '_thread',) def _start(self): - pr, pw = os.pipe() - self._files = {} - self._files['pipe_read'] = pr - self._files['pipe_write'] = pw - for f in self._files.values(): - fcntl.fcntl(f, fcntl.F_SETFL, - fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_id = self.scheduler.register(self._files['pipe_read'], - self.scheduler.IO_IN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: threading_mod = dummy_threading self._thread = threading_mod.Thread(target=self._run_lock) + self._thread.daemon = True self._thread.start() def _run_lock(self): self._lock_obj = lockfile(self.path, wantnewlockfile=True) - os.write(self._files['pipe_write'], b'\0') - - def _output_handler(self, f, event): - buf = None - if event & self.scheduler.IO_IN: - try: - buf = os.read(self._files['pipe_read'], self._bufsize) - except OSError as e: - if e.errno not in (errno.EAGAIN,): - raise - if buf: - self._unregister() - self.returncode = os.EX_OK - self.wait() + # Thread-safe callback to EventLoop + self.scheduler.idle_add(self._run_lock_cb) - return True + def _run_lock_cb(self): + self._unregister() + self.returncode = os.EX_OK + self.wait() + return False def _cancel(self): # There's currently no way to force thread termination. @@ -163,15 +146,6 @@ class _LockThread(AbstractPollTask): self._thread.join() self._thread = None - if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) - self._reg_id = None - - if self._files is not None: - for f in self._files.values(): - os.close(f) - self._files = None - class _LockProcess(AbstractPollTask): """ This uses the portage.locks module to acquire a lock asynchronously, @@ -190,16 +164,28 @@ class _LockProcess(AbstractPollTask): self._files = {} self._files['pipe_in'] = in_pr self._files['pipe_out'] = out_pw + fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_id = self.scheduler.register(in_pr, + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(in_pr, fcntl.F_SETFD, + fcntl.fcntl(in_pr, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch(in_pr, self.scheduler.IO_IN, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[portage._python_interpreter, os.path.join(portage._bin_path, 'lock-helper.py'), self.path], env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path), - fd_pipes={0:out_pr, 1:in_pw, 2:sys.stderr.fileno()}, + fd_pipes={0:out_pr, 1:in_pw, 2:sys.__stderr__.fileno()}, scheduler=self.scheduler) self._proc.addExitListener(self._proc_exit) self._proc.start() @@ -273,7 +259,7 @@ class _LockProcess(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo Binary files differindex 5f3cfbb..5b9031c 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.py b/portage_with_autodep/pym/_emerge/AsynchronousTask.py index 7a193ce..da58261 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousTask.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.py @@ -60,6 +60,20 @@ class AsynchronousTask(SlotObject): def _wait(self): return self.returncode + def _async_wait(self): + """ + For cases where _start exits synchronously, this method is a + convenient way to trigger an asynchronous call to self.wait() + (in order to notify exit listeners), avoiding excessive event + loop recursion (or stack overflow) that synchronous calling of + exit listeners can cause. This method is thread-safe. + """ + self.scheduler.idle_add(self._async_wait_cb) + + def _async_wait_cb(self): + self.wait() + return False + def cancel(self): """ Cancel the task, but do not wait for exit status. If asynchronous exit diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo Binary files differindex b8d67ea..f679452 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AtomArg.py b/portage_with_autodep/pym/_emerge/AtomArg.py index a929b43..343d7aa 100644 --- a/portage_with_autodep/pym/_emerge/AtomArg.py +++ b/portage_with_autodep/pym/_emerge/AtomArg.py @@ -1,10 +1,13 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage._sets.base import InternalPackageSet from _emerge.DependencyArg import DependencyArg class AtomArg(DependencyArg): + + __slots__ = ('atom', 'pset') + def __init__(self, atom=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.atom = atom diff --git a/portage_with_autodep/pym/_emerge/AtomArg.pyo b/portage_with_autodep/pym/_emerge/AtomArg.pyo Binary files differindex b8f59cf..8d9fe2a 100644 --- a/portage_with_autodep/pym/_emerge/AtomArg.pyo +++ b/portage_with_autodep/pym/_emerge/AtomArg.pyo diff --git a/portage_with_autodep/pym/_emerge/Binpkg.py b/portage_with_autodep/pym/_emerge/Binpkg.py index ea8a1ad..a740efd 100644 --- a/portage_with_autodep/pym/_emerge/Binpkg.py +++ b/portage_with_autodep/pym/_emerge/Binpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildPhase import EbuildPhase @@ -298,6 +298,7 @@ class Binpkg(CompositeTask): extractor = BinpkgExtractorAsync(background=self.background, env=self.settings.environ(), + features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), @@ -328,11 +329,13 @@ class Binpkg(CompositeTask): self.wait() return + env = self.settings.environ() + env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"]], - background=self.background, env=self.settings.environ(), + background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) diff --git a/portage_with_autodep/pym/_emerge/Binpkg.pyo b/portage_with_autodep/pym/_emerge/Binpkg.pyo Binary files differindex 4499b9d..7b6472d 100644 --- a/portage_with_autodep/pym/_emerge/Binpkg.pyo +++ b/portage_with_autodep/pym/_emerge/Binpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo Binary files differindex 21c2e13..a86a949 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py index f25cbf9..be74c2f 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py @@ -1,23 +1,31 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SpawnProcess import SpawnProcess import portage import signal +import subprocess class BinpkgExtractorAsync(SpawnProcess): - __slots__ = ("image_dir", "pkg", "pkg_path") + __slots__ = ("features", "image_dir", "pkg", "pkg_path") _shell_binary = portage.const.BASH_BINARY def _start(self): + tar_options = "" + if "xattr" in self.features: + process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = process.communicate()[0] + if b"--xattrs" in output: + tar_options = "--xattrs" + # Add -q to bzip2 opts, in order to avoid "trailing garbage after # EOF ignored" warning messages due to xpak trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with # assert_sigpipe_ok() that's used by the ebuild unpack() helper. self.args = [self._shell_binary, "-c", - ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp -C %s -f - ; " + \ + ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \ "p=(${PIPESTATUS[@]}) ; " + \ "if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \ @@ -25,6 +33,7 @@ class BinpkgExtractorAsync(SpawnProcess): "echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \ "exit 0 ;") % \ (portage._shell_quote(self.pkg_path), + tar_options, portage._shell_quote(self.image_dir))] SpawnProcess._start(self) diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo Binary files differindex f8498f7..83d4d9c 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py index f415e2e..543881e 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousLock import AsynchronousLock @@ -63,7 +63,7 @@ class BinpkgFetcher(SpawnProcess): if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return protocol = urllib_parse_urlparse(uri)[0] @@ -80,6 +80,12 @@ class BinpkgFetcher(SpawnProcess): "FILE" : os.path.basename(pkg_path) } + for k in ("PORTAGE_SSH_OPTS",): + try: + fcmd_vars[k] = settings[k] + except KeyError: + pass + fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] @@ -91,9 +97,9 @@ class BinpkgFetcher(SpawnProcess): # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stdout.fileno()) + fd_pipes.setdefault(0, portage._get_stdin().fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env @@ -104,7 +110,7 @@ class BinpkgFetcher(SpawnProcess): def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" - if self.background or not sys.stdout.isatty(): + if self.background or not sys.__stdout__.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo Binary files differindex 482e55e..1514fb9 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo Binary files differindex c890cac..cfe9e45 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgVerifier.py b/portage_with_autodep/pym/_emerge/BinpkgVerifier.py index 0052967..2c69792 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgVerifier.py +++ b/portage_with_autodep/pym/_emerge/BinpkgVerifier.py @@ -1,75 +1,120 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.AsynchronousTask import AsynchronousTask -from portage.util import writemsg +import errno import io import sys + +from _emerge.CompositeTask import CompositeTask import portage from portage import os +from portage.checksum import (_apply_hash_filter, + _filter_unaccelarated_hashes, _hash_filter) +from portage.output import EOutput +from portage.util._async.FileDigester import FileDigester from portage.package.ebuild.fetch import _checksum_failure_temp_file -class BinpkgVerifier(AsynchronousTask): - __slots__ = ("logfile", "pkg", "scheduler") +class BinpkgVerifier(CompositeTask): + __slots__ = ("logfile", "pkg", "_digests", "_pkg_path") def _start(self): - """ - Note: Unlike a normal AsynchronousTask.start() method, - this one does all work is synchronously. The returncode - attribute will be set before it returns. - """ - - pkg = self.pkg - root_config = pkg.root_config - bintree = root_config.trees["bintree"] - rval = os.EX_OK + + bintree = self.pkg.root_config.trees["bintree"] + digests = bintree._get_digests(self.pkg) + if "size" not in digests: + self.returncode = os.EX_OK + self._async_wait() + return + + digests = _filter_unaccelarated_hashes(digests) + hash_filter = _hash_filter( + bintree.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if not hash_filter.transparent: + digests = _apply_hash_filter(digests, hash_filter) + + self._digests = digests + self._pkg_path = bintree.getname(self.pkg.cpv) + + try: + size = os.stat(self._pkg_path).st_size + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + self.scheduler.output(("!!! Fetching Binary failed " + "for '%s'\n") % self.pkg.cpv, log_path=self.logfile, + background=self.background) + self.returncode = 1 + self._async_wait() + return + else: + if size != digests["size"]: + self._digest_exception("size", size, digests["size"]) + self.returncode = 1 + self._async_wait() + return + + self._start_task(FileDigester(file_path=self._pkg_path, + hash_names=(k for k in digests if k != "size"), + background=self.background, logfile=self.logfile, + scheduler=self.scheduler), + self._digester_exit) + + def _digester_exit(self, digester): + + if self._default_exit(digester) != os.EX_OK: + self.wait() + return + + for hash_name in digester.hash_names: + if digester.digests[hash_name] != self._digests[hash_name]: + self._digest_exception(hash_name, + digester.digests[hash_name], self._digests[hash_name]) + self.returncode = 1 + self.wait() + return + + if self.pkg.root_config.settings.get("PORTAGE_QUIET") != "1": + self._display_success() + + self.returncode = os.EX_OK + self.wait() + + def _display_success(self): stdout_orig = sys.stdout stderr_orig = sys.stderr global_havecolor = portage.output.havecolor out = io.StringIO() - file_exists = True try: sys.stdout = out sys.stderr = out if portage.output.havecolor: portage.output.havecolor = not self.background - try: - bintree.digestCheck(pkg) - except portage.exception.FileNotFound: - writemsg("!!! Fetching Binary failed " + \ - "for '%s'\n" % pkg.cpv, noiselevel=-1) - rval = 1 - file_exists = False - except portage.exception.DigestException as e: - writemsg("\n!!! Digest verification failed:\n", - noiselevel=-1) - writemsg("!!! %s\n" % e.value[0], - noiselevel=-1) - writemsg("!!! Reason: %s\n" % e.value[1], - noiselevel=-1) - writemsg("!!! Got: %s\n" % e.value[2], - noiselevel=-1) - writemsg("!!! Expected: %s\n" % e.value[3], - noiselevel=-1) - rval = 1 - if rval == os.EX_OK: - pass - elif file_exists: - pkg_path = bintree.getname(pkg.cpv) - head, tail = os.path.split(pkg_path) - temp_filename = _checksum_failure_temp_file(head, tail) - writemsg("File renamed to '%s'\n" % (temp_filename,), - noiselevel=-1) + + eout = EOutput() + eout.ebegin("%s %s ;-)" % (os.path.basename(self._pkg_path), + " ".join(sorted(self._digests)))) + eout.eend(0) + finally: sys.stdout = stdout_orig sys.stderr = stderr_orig portage.output.havecolor = global_havecolor - msg = out.getvalue() - if msg: - self.scheduler.output(msg, log_path=self.logfile, - background=self.background) + self.scheduler.output(out.getvalue(), log_path=self.logfile, + background=self.background) - self.returncode = rval - self.wait() + def _digest_exception(self, name, value, expected): + + head, tail = os.path.split(self._pkg_path) + temp_filename = _checksum_failure_temp_file(head, tail) + self.scheduler.output(( + "\n!!! Digest verification failed:\n" + "!!! %s\n" + "!!! Reason: Failed on %s verification\n" + "!!! Got: %s\n" + "!!! Expected: %s\n" + "File renamed to '%s'\n") % + (self._pkg_path, name, value, expected, temp_filename), + log_path=self.logfile, + background=self.background) diff --git a/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo Binary files differindex 21f770e..1c84b08 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo diff --git a/portage_with_autodep/pym/_emerge/Blocker.pyo b/portage_with_autodep/pym/_emerge/Blocker.pyo Binary files differindex b9e56bc..8905faf 100644 --- a/portage_with_autodep/pym/_emerge/Blocker.pyo +++ b/portage_with_autodep/pym/_emerge/Blocker.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.py b/portage_with_autodep/pym/_emerge/BlockerCache.py index fce81f8..53342d6 100644 --- a/portage_with_autodep/pym/_emerge/BlockerCache.py +++ b/portage_with_autodep/pym/_emerge/BlockerCache.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -62,7 +62,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping): self._cache_data = mypickle.load() f.close() del f - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: if isinstance(e, EnvironmentError) and \ getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass @@ -126,9 +128,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping): self._modified.clear() def flush(self): - """If the current user has permission and the internal blocker cache + """If the current user has permission and the internal blocker cache has been updated, save it to disk and mark it unmodified. This is called - by emerge after it has proccessed blockers for all installed packages. + by emerge after it has processed blockers for all installed packages. Currently, the cache is only written if the user has superuser privileges (since that's required to obtain a lock), but all users have read access and benefit from faster blocker lookups (as long as diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.pyo b/portage_with_autodep/pym/_emerge/BlockerCache.pyo Binary files differindex 41554e1..632df92 100644 --- a/portage_with_autodep/pym/_emerge/BlockerCache.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerCache.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.py b/portage_with_autodep/pym/_emerge/BlockerDB.py index 459affd..8bb8f5f 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDB.py +++ b/portage_with_autodep/pym/_emerge/BlockerDB.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -9,6 +9,7 @@ from portage import digraph from portage._sets.base import InternalPackageSet from _emerge.BlockerCache import BlockerCache +from _emerge.Package import Package from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice if sys.hexversion >= 0x3000000: @@ -38,7 +39,7 @@ class BlockerDB(object): """ blocker_cache = BlockerCache(None, self._vartree.dbapi) - dep_keys = ["RDEPEND", "PDEPEND"] + dep_keys = Package._runtime_keys settings = self._vartree.settings stale_cache = set(blocker_cache) fake_vartree = self._fake_vartree @@ -50,7 +51,7 @@ class BlockerDB(object): stale_cache.discard(inst_pkg.cpv) cached_blockers = blocker_cache.get(inst_pkg.cpv) if cached_blockers is not None and \ - cached_blockers.counter != long(inst_pkg.metadata["COUNTER"]): + cached_blockers.counter != inst_pkg.counter: cached_blockers = None if cached_blockers is not None: blocker_atoms = cached_blockers.atoms @@ -71,9 +72,8 @@ class BlockerDB(object): blocker_atoms = [atom for atom in atoms \ if atom.startswith("!")] blocker_atoms.sort() - counter = long(inst_pkg.metadata["COUNTER"]) blocker_cache[inst_pkg.cpv] = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(inst_pkg.counter, blocker_atoms) for cpv in stale_cache: del blocker_cache[cpv] blocker_cache.flush() @@ -92,7 +92,7 @@ class BlockerDB(object): blocking_pkgs.update(blocker_parents.parent_nodes(atom)) # Check for blockers in the other direction. - depstr = " ".join(new_pkg.metadata[k] for k in dep_keys) + depstr = " ".join(new_pkg._metadata[k] for k in dep_keys) success, atoms = portage.dep_check(depstr, vardb, settings, myuse=new_pkg.use.enabled, trees=dep_check_trees, myroot=new_pkg.root) diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.pyo b/portage_with_autodep/pym/_emerge/BlockerDB.pyo Binary files differindex dfab0aa..15ab6cd 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDB.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerDB.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo Binary files differindex c3b554c..c998728 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.py b/portage_with_autodep/pym/_emerge/CompositeTask.py index 3e43478..40cf859 100644 --- a/portage_with_autodep/pym/_emerge/CompositeTask.py +++ b/portage_with_autodep/pym/_emerge/CompositeTask.py @@ -142,6 +142,10 @@ class CompositeTask(AsynchronousTask): a task. """ + try: + task.scheduler = self.scheduler + except AttributeError: + pass task.addExitListener(exit_handler) self._current_task = task task.start() diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.pyo b/portage_with_autodep/pym/_emerge/CompositeTask.pyo Binary files differindex adc8cae..f41c565 100644 --- a/portage_with_autodep/pym/_emerge/CompositeTask.pyo +++ b/portage_with_autodep/pym/_emerge/CompositeTask.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriority.py b/portage_with_autodep/pym/_emerge/DepPriority.py index 3c2256a..34fdb48 100644 --- a/portage_with_autodep/pym/_emerge/DepPriority.py +++ b/portage_with_autodep/pym/_emerge/DepPriority.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractDepPriority import AbstractDepPriority @@ -16,31 +16,38 @@ class DepPriority(AbstractDepPriority): Attributes Hardness - buildtime 0 - runtime -1 - runtime_post -2 - optional -3 - (none of the above) -4 + buildtime_slot_op 0 + buildtime -1 + runtime -2 + runtime_post -3 + optional -4 + (none of the above) -5 """ if self.optional: - return -3 - if self.buildtime: + return -4 + if self.buildtime_slot_op: return 0 - if self.runtime: + if self.buildtime: return -1 - if self.runtime_post: + if self.runtime: return -2 - return -4 + if self.runtime_post: + return -3 + return -5 def __str__(self): if self.ignored: return "ignored" if self.optional: return "optional" + if self.buildtime_slot_op: + return "buildtime_slot_op" if self.buildtime: return "buildtime" + if self.runtime_slot_op: + return "runtime_slot_op" if self.runtime: return "runtime" if self.runtime_post: diff --git a/portage_with_autodep/pym/_emerge/DepPriority.pyo b/portage_with_autodep/pym/_emerge/DepPriority.pyo Binary files differindex 4028a36..ade8cdd 100644 --- a/portage_with_autodep/pym/_emerge/DepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/DepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo Binary files differindex 5e0f710..a255f1d 100644 --- a/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo +++ b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py index edb29df..391f540 100644 --- a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py +++ b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.DepPriority import DepPriority @@ -7,17 +7,18 @@ class DepPrioritySatisfiedRange(object): DepPriority Index Category not satisfied and buildtime HARD - not satisfied and runtime 6 MEDIUM - not satisfied and runtime_post 5 MEDIUM_SOFT + not satisfied and runtime 7 MEDIUM + not satisfied and runtime_post 6 MEDIUM_SOFT + satisfied and buildtime_slot_op 5 SOFT satisfied and buildtime 4 SOFT satisfied and runtime 3 SOFT satisfied and runtime_post 2 SOFT optional 1 SOFT (none of the above) 0 NONE """ - MEDIUM = 6 - MEDIUM_SOFT = 5 - SOFT = 4 + MEDIUM = 7 + MEDIUM_SOFT = 6 + SOFT = 5 NONE = 0 @classmethod @@ -50,6 +51,16 @@ class DepPrioritySatisfiedRange(object): def _ignore_satisfied_buildtime(cls, priority): if priority.__class__ is not DepPriority: return False + if priority.optional: + return True + if priority.buildtime_slot_op: + return False + return bool(priority.satisfied) + + @classmethod + def _ignore_satisfied_buildtime_slot_op(cls, priority): + if priority.__class__ is not DepPriority: + return False return bool(priority.optional or \ priority.satisfied) @@ -80,6 +91,7 @@ DepPrioritySatisfiedRange.ignore_priority = ( DepPrioritySatisfiedRange._ignore_satisfied_runtime_post, DepPrioritySatisfiedRange._ignore_satisfied_runtime, DepPrioritySatisfiedRange._ignore_satisfied_buildtime, + DepPrioritySatisfiedRange._ignore_satisfied_buildtime_slot_op, DepPrioritySatisfiedRange._ignore_runtime_post, DepPrioritySatisfiedRange._ignore_runtime ) diff --git a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo Binary files differindex 5309bcd..7f95d84 100644 --- a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo +++ b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo diff --git a/portage_with_autodep/pym/_emerge/Dependency.py b/portage_with_autodep/pym/_emerge/Dependency.py index c2d36b2..2ec860f 100644 --- a/portage_with_autodep/pym/_emerge/Dependency.py +++ b/portage_with_autodep/pym/_emerge/Dependency.py @@ -6,7 +6,7 @@ from _emerge.DepPriority import DepPriority class Dependency(SlotObject): __slots__ = ("atom", "blocker", "child", "depth", - "parent", "onlydeps", "priority", "root", + "parent", "onlydeps", "priority", "root", "want_update", "collapsed_parent", "collapsed_priority") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) diff --git a/portage_with_autodep/pym/_emerge/Dependency.pyo b/portage_with_autodep/pym/_emerge/Dependency.pyo Binary files differindex f53e0ed..b1428d5 100644 --- a/portage_with_autodep/pym/_emerge/Dependency.pyo +++ b/portage_with_autodep/pym/_emerge/Dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/DependencyArg.py b/portage_with_autodep/pym/_emerge/DependencyArg.py index 861d837..29a0072 100644 --- a/portage_with_autodep/pym/_emerge/DependencyArg.py +++ b/portage_with_autodep/pym/_emerge/DependencyArg.py @@ -1,13 +1,26 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys -from portage import _encodings, _unicode_encode, _unicode_decode +from portage import _encodings, _unicode_encode class DependencyArg(object): - def __init__(self, arg=None, root_config=None): + + __slots__ = ('arg', 'force_reinstall', 'internal', 'reset_depth', 'root_config') + + def __init__(self, arg=None, force_reinstall=False, internal=False, + reset_depth=True, root_config=None): + """ + Use reset_depth=False for special arguments that should not interact + with depth calculations (see the emerge --deep=DEPTH option). + """ self.arg = arg + self.force_reinstall = force_reinstall + self.internal = internal + self.reset_depth = reset_depth self.root_config = root_config def __eq__(self, other): @@ -20,10 +33,10 @@ class DependencyArg(object): return hash((self.arg, self.root_config.root)) def __str__(self): - # Force unicode format string for python-2.x safety, + # Use unicode_literals format string for python-2.x safety, # ensuring that self.arg.__unicode__() is used # when necessary. - return _unicode_decode("%s") % (self.arg,) + return "%s" % (self.arg,) if sys.hexversion < 0x3000000: diff --git a/portage_with_autodep/pym/_emerge/DependencyArg.pyo b/portage_with_autodep/pym/_emerge/DependencyArg.pyo Binary files differindex 916a762..536670c 100644 --- a/portage_with_autodep/pym/_emerge/DependencyArg.pyo +++ b/portage_with_autodep/pym/_emerge/DependencyArg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo Binary files differindex 2acfc87..3190c6a 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.py b/portage_with_autodep/pym/_emerge/EbuildBuild.py index 5a48f8e..f680434 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuild.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildExecuter import EbuildExecuter @@ -12,11 +12,14 @@ from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator from _emerge.EventsLogger import EventsLogger from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.TaskSequence import TaskSequence + from portage.util import writemsg import portage from portage import os from portage.output import colorize from portage.package.ebuild.digestcheck import digestcheck +from portage.package.ebuild.digestgen import digestgen from portage.package.ebuild.doebuild import _check_temp_dir from portage.package.ebuild._spawn_nofetch import spawn_nofetch @@ -37,7 +40,7 @@ class EbuildBuild(CompositeTask): if rval != os.EX_OK: self.returncode = rval self._current_task = None - self.wait() + self._async_wait() return root_config = pkg.root_config @@ -62,7 +65,7 @@ class EbuildBuild(CompositeTask): if not self._check_manifest(): self.returncode = 1 self._current_task = None - self.wait() + self._async_wait() return prefetcher = self.prefetcher @@ -93,7 +96,8 @@ class EbuildBuild(CompositeTask): success = True settings = self.settings - if 'strict' in settings.features: + if 'strict' in settings.features and \ + 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' @@ -162,6 +166,10 @@ class EbuildBuild(CompositeTask): if self.returncode != os.EX_OK: portdb = self.pkg.root_config.trees[self._tree].dbapi spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) + elif 'digest' in self.settings.features: + if not digestgen(mysettings=self.settings, + myportdb=self.pkg.root_config.trees[self._tree].dbapi): + self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): @@ -264,7 +272,7 @@ class EbuildBuild(CompositeTask): "depcheckstrict" in self.settings["FEATURES"]: # Lets start a log listening server temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) - + if "depcheckstrict" not in self.settings["FEATURES"]: # use default filter_proc self.logserver=EventsLogger(socket_dir=temp_path) @@ -273,11 +281,11 @@ class EbuildBuild(CompositeTask): "This may take some time\n") filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) filter_proc=filter_gen.get_filter_proc() - self.logserver=EventsLogger(socket_dir=temp_path, + self.logserver=EventsLogger(socket_dir=temp_path, filter_proc=filter_proc) - + self.logserver.start() - + # Copy socket path to LOG_SOCKET environment variable env=self.settings.configdict["pkg"] env['LOG_SOCKET'] = self.logserver.socket_name @@ -291,15 +299,13 @@ class EbuildBuild(CompositeTask): env=self.settings.configdict["pkg"] if 'LOG_SOCKET' in env: del env['LOG_SOCKET'] - + events=self.logserver.stop() self.logserver=None analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) analyser.display() # show the analyse #import pdb; pdb.set_trace() - - def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch @@ -308,8 +314,8 @@ class EbuildBuild(CompositeTask): # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). - if 'fetch' not in self.pkg.metadata.restrict and \ - 'nofetch' not in self.pkg.metadata.defined_phases: + if 'fetch' not in self.pkg.restrict and \ + 'nofetch' not in self.pkg.defined_phases: self._unlock_builddir() self.wait() return @@ -348,10 +354,20 @@ class EbuildBuild(CompositeTask): self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) - packager = EbuildBinpkg(background=self.background, pkg=self.pkg, - scheduler=self.scheduler, settings=self.settings) + binpkg_tasks = TaskSequence() + requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() + for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: + if pkg_fmt in requested_binpkg_formats: + if pkg_fmt == "rpm": + binpkg_tasks.add(EbuildPhase(background=self.background, + phase="rpm", scheduler=self.scheduler, + settings=self.settings)) + else: + binpkg_tasks.add(EbuildBinpkg(background=self.background, + pkg=self.pkg, scheduler=self.scheduler, + settings=self.settings)) - self._start_task(packager, self._buildpkg_exit) + self._start_task(binpkg_tasks, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.pyo b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo Binary files differindex 19d913c..78bf68d 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuild.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py index 9773bd7..58905c2 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py @@ -7,7 +7,6 @@ import portage from portage import os from portage.exception import PortageException from portage.util.SlotObject import SlotObject -import errno class EbuildBuildDir(SlotObject): @@ -60,7 +59,7 @@ class EbuildBuildDir(SlotObject): builddir_lock.wait() self._assert_lock(builddir_lock) self._lock_obj = builddir_lock - self.settings['PORTAGE_BUILDIR_LOCKED'] = '1' + self.settings['PORTAGE_BUILDDIR_LOCKED'] = '1' finally: self.locked = self._lock_obj is not None catdir_lock.unlock() @@ -92,16 +91,14 @@ class EbuildBuildDir(SlotObject): self._lock_obj.unlock() self._lock_obj = None self.locked = False - self.settings.pop('PORTAGE_BUILDIR_LOCKED', None) + self.settings.pop('PORTAGE_BUILDDIR_LOCKED', None) catdir_lock = AsynchronousLock(path=self._catdir, scheduler=self.scheduler) catdir_lock.start() if catdir_lock.wait() == os.EX_OK: try: os.rmdir(self._catdir) - except OSError as e: - if e.errno not in (errno.ENOENT, - errno.ENOTEMPTY, errno.EEXIST, errno.EPERM): - raise + except OSError: + pass finally: catdir_lock.unlock() diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo Binary files differindex 2846579..290f8d9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.py b/portage_with_autodep/pym/_emerge/EbuildExecuter.py index fd663a4..5587d4e 100644 --- a/portage_with_autodep/pym/_emerge/EbuildExecuter.py +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.py @@ -16,16 +16,7 @@ class EbuildExecuter(CompositeTask): _phases = ("prepare", "configure", "compile", "test", "install") - _live_eclasses = frozenset([ - "bzr", - "cvs", - "darcs", - "git", - "git-2", - "mercurial", - "subversion", - "tla", - ]) + _live_eclasses = portage.const.LIVE_ECLASSES def _start(self): pkg = self.pkg @@ -83,7 +74,7 @@ class EbuildExecuter(CompositeTask): pkg = self.pkg phases = self._phases - eapi = pkg.metadata["EAPI"] + eapi = pkg.eapi if not eapi_has_src_prepare_and_src_configure(eapi): # skip src_prepare and src_configure phases = phases[2:] diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo Binary files differindex 592a0c9..21fc3d3 100644 --- a/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.py b/portage_with_autodep/pym/_emerge/EbuildFetcher.py index c0a7fdd..d98d007 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetcher.py +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.py @@ -1,23 +1,22 @@ # Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import traceback - -from _emerge.SpawnProcess import SpawnProcess import copy import io -import signal import sys + import portage from portage import os from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode +from portage.checksum import _hash_filter from portage.elog.messages import eerror from portage.package.ebuild.fetch import _check_distfile, fetch +from portage.util._async.ForkProcess import ForkProcess from portage.util._pty import _create_pty_or_pipe -class EbuildFetcher(SpawnProcess): +class EbuildFetcher(ForkProcess): __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", "pkg", "prefetch") + \ @@ -57,6 +56,9 @@ class EbuildFetcher(SpawnProcess): if st.st_size != expected_size: return False + hash_filter = _hash_filter(settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None stdout_orig = sys.stdout stderr_orig = sys.stderr global_havecolor = portage.output.havecolor @@ -78,7 +80,7 @@ class EbuildFetcher(SpawnProcess): break continue ok, st = _check_distfile(os.path.join(distdir, filename), - mydigests, eout, show_errors=False) + mydigests, eout, show_errors=False, hash_filter=hash_filter) if not ok: success = False break @@ -115,13 +117,13 @@ class EbuildFetcher(SpawnProcess): msg_lines.append(msg) self._eerror(msg_lines) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return if not uri_map: # Nothing to fetch. self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return settings = self.config_pool.allocate() @@ -133,7 +135,7 @@ class EbuildFetcher(SpawnProcess): self._prefetch_size_ok(uri_map, settings, ebuild_path): self.config_pool.deallocate(settings) self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return nocolor = settings.get("NOCOLOR") @@ -148,7 +150,7 @@ class EbuildFetcher(SpawnProcess): settings["NOCOLOR"] = nocolor self._settings = settings - SpawnProcess._start(self) + ForkProcess._start(self) # Free settings now since it's no longer needed in # this process (the subprocess has a private copy). @@ -156,48 +158,20 @@ class EbuildFetcher(SpawnProcess): settings = None self._settings = None - def _spawn(self, args, fd_pipes=None, **kwargs): - """ - Fork a subprocess, apply local settings, and call fetch(). - """ - - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - portage.process.spawned_pids.append(pid) - return [pid] - - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers in order to avoid problems - # killing subprocesses as reported in bug #353239. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - + def _run(self): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 - allow_missing = self._get_manifest().allow_missing - try: - if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, - digests=copy.deepcopy(self._get_digests()), - allow_missing_digests=allow_missing): - rval = os.EX_OK - except SystemExit: - raise - except: - traceback.print_exc() - finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + allow_missing = self._get_manifest().allow_missing or \ + 'digest' in self._settings.features + if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, + digests=copy.deepcopy(self._get_digests()), + allow_missing_digests=allow_missing): + rval = os.EX_OK + return rval def _get_ebuild_path(self): if self.ebuild_path is not None: @@ -297,7 +271,7 @@ class EbuildFetcher(SpawnProcess): self.scheduler.output(msg, log_path=self.logfile) def _set_returncode(self, wait_retval): - SpawnProcess._set_returncode(self, wait_retval) + ForkProcess._set_returncode(self, wait_retval) # Collect elog messages that might have been # created by the pkg_nofetch phase. # Skip elog messages for prefetch, in order to avoid duplicates. diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo Binary files differindex ddc92d1..e87abd9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo Binary files differindex c54a1db..947ab78 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo Binary files differindex 7a9588f..fbc4edc 100644 --- a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMerge.pyo b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo Binary files differindex 662c681..b281450 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMerge.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py index c2d3747..bbb1ca9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -6,12 +6,14 @@ import sys from portage.cache.mappings import slot_dict_class import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.package.ebuild._eapi_invalid:eapi_invalid', + 'portage.package.ebuild._metadata_invalid:eapi_invalid', ) from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.dep import extract_unpack_dependencies +from portage.eapi import eapi_has_automatic_unpack_dependencies import errno import fcntl @@ -25,12 +27,11 @@ class EbuildMetadataPhase(SubProcess): """ __slots__ = ("cpv", "eapi_supported", "ebuild_hash", "fd_pipes", - "metadata", "portdb", "repo_path", "settings") + \ + "metadata", "portdb", "repo_path", "settings", "write_auxdb") + \ ("_eapi", "_eapi_lineno", "_raw_metadata",) _file_names = ("ebuild",) _files_dict = slot_dict_class(_file_names, prefix="") - _metadata_fd = 9 def _start(self): ebuild_path = self.ebuild_hash.location @@ -49,14 +50,14 @@ class EbuildMetadataPhase(SubProcess): # An empty EAPI setting is invalid. self._eapi_invalid(None) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return self.eapi_supported = portage.eapi_is_supported(parsed_eapi) if not self.eapi_supported: self.metadata = {"EAPI": parsed_eapi} self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return settings = self.settings @@ -74,28 +75,41 @@ class EbuildMetadataPhase(SubProcess): null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() + if fd in stdout_filenos: + sys.__stdout__.flush() + sys.__stderr__.flush() + break self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() + fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fd_pipes[self._metadata_fd] = slave_fd + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(master_fd, fcntl.F_SETFD, + fcntl.fcntl(master_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + fd_pipes[slave_fd] = slave_fd + settings["PORTAGE_PIPE_FD"] = str(slave_fd) self._raw_metadata = [] files.ebuild = master_fd - self._reg_id = self.scheduler.register(files.ebuild, + self._reg_id = self.scheduler.io_add_watch(files.ebuild, self._registered_events, self._output_handler) self._registered = True @@ -103,6 +117,7 @@ class EbuildMetadataPhase(SubProcess): settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) + settings.pop("PORTAGE_PIPE_FD", None) os.close(slave_fd) null_input.close() @@ -111,11 +126,10 @@ class EbuildMetadataPhase(SubProcess): # doebuild failed before spawning self._unregister() self._set_returncode((self.pid, retval << 8)) - self.wait() + self._async_wait() return self.pid = retval[0] - portage.process.spawned_pids.remove(self.pid) def _output_handler(self, fd, event): @@ -141,8 +155,7 @@ class EbuildMetadataPhase(SubProcess): def _set_returncode(self, wait_retval): SubProcess._set_returncode(self, wait_retval) # self._raw_metadata is None when _start returns - # early due to an unsupported EAPI detected with - # FEATURES=parse-eapi-ebuild-head + # early due to an unsupported EAPI if self.returncode == os.EX_OK and \ self._raw_metadata is not None: metadata_lines = _unicode_decode(b''.join(self._raw_metadata), @@ -163,8 +176,7 @@ class EbuildMetadataPhase(SubProcess): if (not metadata["EAPI"] or self.eapi_supported) and \ metadata["EAPI"] != parsed_eapi: self._eapi_invalid(metadata) - if 'parse-eapi-ebuild-head' in self.settings.features: - metadata_valid = False + metadata_valid = False if metadata_valid: # Since we're supposed to be able to efficiently obtain the @@ -181,8 +193,18 @@ class EbuildMetadataPhase(SubProcess): metadata["_eclasses_"] = {} metadata.pop("INHERITED", None) - self.portdb._write_cache(self.cpv, - self.repo_path, metadata, self.ebuild_hash) + if eapi_has_automatic_unpack_dependencies(metadata["EAPI"]): + repo = self.portdb.repositories.get_name_for_location(self.repo_path) + unpackers = self.settings.unpack_dependencies.get(repo, {}).get(metadata["EAPI"], {}) + unpack_dependencies = extract_unpack_dependencies(metadata["SRC_URI"], unpackers) + if unpack_dependencies: + metadata["DEPEND"] += (" " if metadata["DEPEND"] else "") + unpack_dependencies + + # If called by egencache, this cache write is + # undesirable when metadata-transfer is disabled. + if self.write_auxdb is not False: + self.portdb._write_cache(self.cpv, + self.repo_path, metadata, self.ebuild_hash) else: metadata = {"EAPI": metadata["EAPI"]} self.metadata = metadata diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo Binary files differindex fcc0874..20c9574 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py b/portage_with_autodep/pym/_emerge/EbuildPhase.py index 36ca8b0..0916d73 100644 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gzip @@ -11,16 +11,26 @@ from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask +from portage.package.ebuild.prepare_build_dirs import _prepare_workdir from portage.util import writemsg -from portage.xml.metadata import MetaDataXML + +try: + from portage.xml.metadata import MetaDataXML +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 + MetaDataXML = None + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.elog:messages@elog_messages', 'portage.package.ebuild.doebuild:_check_build_log,' + \ '_post_phase_cmds,_post_phase_userpriv_perms,' + \ - '_post_src_install_chost_fix,' + \ '_post_src_install_soname_symlinks,' + \ '_post_src_install_uid_fix,_postinst_bsdflags,' + \ + '_post_src_install_write_metadata,' + \ '_preinst_bsdflags' ) from portage import os @@ -29,12 +39,12 @@ from portage import _unicode_encode class EbuildPhase(CompositeTask): - __slots__ = ("actionmap", "phase", "settings") + \ + __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \ ("_ebuild_lock",) # FEATURES displayed prior to setup phase _features_display = ( - "ccache", "compressdebug", "depcheck", "depcheckstrict", + "ccache", "compressdebug", "depcheck", "depcheckstrict", "distcc", "distcc-pump", "fakeroot", "installsources", "keeptemp", "keepwork", "nostrip", "preserve-libs", "sandbox", "selinux", "sesandbox", @@ -72,7 +82,7 @@ class EbuildPhase(CompositeTask): maint_str = "" upstr_str = "" metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml") - if os.path.isfile(metadata_xml_path): + if MetaDataXML is not None and os.path.isfile(metadata_xml_path): herds_path = os.path.join(self.settings['PORTDIR'], 'metadata/herds.xml') try: @@ -148,8 +158,7 @@ class EbuildPhase(CompositeTask): return self._start_ebuild() - def _start_ebuild(self): - + def _get_log_path(self): # Don't open the log file during the clean phase since the # open file can result in an nfs lock on $T/build.log which # prevents the clean phase from removing $T. @@ -157,17 +166,21 @@ class EbuildPhase(CompositeTask): if self.phase not in ("clean", "cleanrm") and \ self.settings.get("PORTAGE_BACKGROUND") != "subprocess": logfile = self.settings.get("PORTAGE_LOG_FILE") + return logfile - fd_pipes = None - if not self.background and self.phase == 'nofetch': - # All the pkg_nofetch output goes to stderr since - # it's considered to be an error message. - fd_pipes = {1 : sys.stderr.fileno()} + def _start_ebuild(self): + + fd_pipes = self.fd_pipes + if fd_pipes is None: + if not self.background and self.phase == 'nofetch': + # All the pkg_nofetch output goes to stderr since + # it's considered to be an error message. + fd_pipes = {1 : sys.__stderr__.fileno()} ebuild_process = EbuildProcess(actionmap=self.actionmap, - background=self.background, fd_pipes=fd_pipes, logfile=logfile, - phase=self.phase, scheduler=self.scheduler, - settings=self.settings) + background=self.background, fd_pipes=fd_pipes, + logfile=self._get_log_path(), phase=self.phase, + scheduler=self.scheduler, settings=self.settings) self._start_task(ebuild_process, self._ebuild_exit) @@ -181,16 +194,21 @@ class EbuildPhase(CompositeTask): if self._default_exit(ebuild_process) != os.EX_OK: if self.phase == "test" and \ "test-fail-continue" in self.settings.features: - pass + # mark test phase as complete (bug #452030) + try: + open(_unicode_encode(os.path.join( + self.settings["PORTAGE_BUILDDIR"], ".tested"), + encoding=_encodings['fs'], errors='strict'), + 'wb').close() + except OSError: + pass else: fail = True if not fail: self.returncode = None - logfile = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - logfile = self.settings.get("PORTAGE_LOG_FILE") + logfile = self._get_log_path() if self.phase == "install": out = io.StringIO() @@ -205,9 +223,16 @@ class EbuildPhase(CompositeTask): settings = self.settings _post_phase_userpriv_perms(settings) - if self.phase == "install": + if self.phase == "unpack": + # Bump WORKDIR timestamp, in case tar gave it a timestamp + # that will interfere with distfiles / WORKDIR timestamp + # comparisons as reported in bug #332217. Also, fix + # ownership since tar can change that too. + os.utime(settings["WORKDIR"], None) + _prepare_workdir(settings) + elif self.phase == "install": out = io.StringIO() - _post_src_install_chost_fix(settings) + _post_src_install_write_metadata(settings) _post_src_install_uid_fix(settings, out) msg = out.getvalue() if msg: @@ -227,8 +252,9 @@ class EbuildPhase(CompositeTask): fd, logfile = tempfile.mkstemp() os.close(fd) post_phase = MiscFunctionsProcess(background=self.background, - commands=post_phase_cmds, logfile=logfile, phase=self.phase, - scheduler=self.scheduler, settings=settings) + commands=post_phase_cmds, fd_pipes=self.fd_pipes, + logfile=logfile, phase=self.phase, scheduler=self.scheduler, + settings=settings) self._start_task(post_phase, self._post_phase_exit) return @@ -303,8 +329,9 @@ class EbuildPhase(CompositeTask): self.returncode = None phase = 'die_hooks' die_hooks = MiscFunctionsProcess(background=self.background, - commands=[phase], phase=phase, - scheduler=self.scheduler, settings=self.settings) + commands=[phase], phase=phase, logfile=self._get_log_path(), + fd_pipes=self.fd_pipes, scheduler=self.scheduler, + settings=self.settings) self._start_task(die_hooks, self._die_hooks_exit) def _die_hooks_exit(self, die_hooks): @@ -323,7 +350,8 @@ class EbuildPhase(CompositeTask): portage.elog.elog_process(self.settings.mycpv, self.settings) phase = "clean" clean_phase = EbuildPhase(background=self.background, - phase=phase, scheduler=self.scheduler, settings=self.settings) + fd_pipes=self.fd_pipes, phase=phase, scheduler=self.scheduler, + settings=self.settings) self._start_task(clean_phase, self._fail_clean_exit) return diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej b/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej deleted file mode 100644 index 0f061da..0000000 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej +++ /dev/null @@ -1,12 +0,0 @@ ---- pym/_emerge/EbuildPhase.py -+++ pym/_emerge/EbuildPhase.py -@@ -33,7 +33,8 @@ - ("_ebuild_lock",) - - # FEATURES displayed prior to setup phase -- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", -+ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", -+ "distcc-pump", "fakeroot", - "installsources", "keeptemp", "keepwork", "nostrip", - "preserve-libs", "sandbox", "selinux", "sesandbox", - "splitdebug", "suidctl", "test", "userpriv", diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo Binary files differindex 4c73313..8c719a5 100644 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildProcess.py b/portage_with_autodep/pym/_emerge/EbuildProcess.py index ce97aff..333ad7b 100644 --- a/portage_with_autodep/pym/_emerge/EbuildProcess.py +++ b/portage_with_autodep/pym/_emerge/EbuildProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -17,5 +17,11 @@ class EbuildProcess(AbstractEbuildProcess): if actionmap is None: actionmap = _spawn_actionmap(self.settings) - return _doebuild_spawn(self.phase, self.settings, - actionmap=actionmap, **kwargs) + if self._dummy_pipe_fd is not None: + self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + + try: + return _doebuild_spawn(self.phase, self.settings, + actionmap=actionmap, **kwargs) + finally: + self.settings.pop("PORTAGE_PIPE_FD", None) diff --git a/portage_with_autodep/pym/_emerge/EbuildProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo Binary files differindex 52f6cdf..7332563 100644 --- a/portage_with_autodep/pym/_emerge/EbuildProcess.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py index e1f682a..26d26fc 100644 --- a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py +++ b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -13,4 +13,10 @@ class EbuildSpawnProcess(AbstractEbuildProcess): __slots__ = ('fakeroot_state', 'spawn_func') def _spawn(self, args, **kwargs): - return self.spawn_func(args, env=self.settings.environ(), **kwargs) + + env = self.settings.environ() + + if self._dummy_pipe_fd is not None: + env["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + + return self.spawn_func(args, env=env, **kwargs) diff --git a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo Binary files differindex 1f3e925..4cfe833 100644 --- a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EventsAnalyser.py b/portage_with_autodep/pym/_emerge/EventsAnalyser.py index 65ece7b..f740cf6 100644 --- a/portage_with_autodep/pym/_emerge/EventsAnalyser.py +++ b/portage_with_autodep/pym/_emerge/EventsAnalyser.py @@ -34,36 +34,36 @@ class PortageUtils: def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): """ Gets current dependencies of a package. Looks in portage db - + :param pkg: name of package - :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] :returns: **set** of packages names """ ret=set() - - pkg = self.get_best_visible_pkg(pkg) + + pkg = self.get_best_visible_pkg(pkg) if not pkg: return ret - + # we found the best visible match in common tree - metadata = dict(zip(self.metadata_keys, + metadata = dict(zip(self.metadata_keys, self.portdbapi.aux_get(pkg, self.metadata_keys))) dep_str = " ".join(metadata[k] for k in dep_type) # the IUSE default are very important for us iuse_defaults=[ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] - + use=self.use.split() - + for u in iuse_defaults: if u not in use: use.append(u) - success, atoms = portage.dep_check(dep_str, None, self.settings, + success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=use, myroot=self.settings["ROOT"], trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) if not success: @@ -74,7 +74,7 @@ class PortageUtils: if not atomname: continue - + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): for pkg in self.vartree.dep_match(unvirt_pkg): ret.add(pkg) @@ -83,12 +83,12 @@ class PortageUtils: # recursive dependency getter def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): - """ - Gets current dependencies of a package on any depth + """ + Gets current dependencies of a package on any depth All dependencies **must** be installed - + :param pkg: name of package - :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] :returns: **set** of packages names """ @@ -97,14 +97,14 @@ class PortageUtils: # get porttree dependencies on the first package - pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) + pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) if not pkg: return ret known_packages=set() unknown_packages=self.get_dep(pkg,dep_type) ret=ret.union(unknown_packages) - + while unknown_packages: p=unknown_packages.pop() if p in known_packages: @@ -114,18 +114,18 @@ class PortageUtils: metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) dep_str = " ".join(metadata[k] for k in dep_type) - + # the IUSE default are very important for us iuse_defaults=[ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] - + use=self.use.split() - + for u in iuse_defaults: if u not in use: use.append(u) - - success, atoms = portage.dep_check(dep_str, None, self.settings, + + success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=use, myroot=self.settings["ROOT"], trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) @@ -136,7 +136,7 @@ class PortageUtils: atomname = self.vartree.dep_bestmatch(atom) if not atomname: continue - + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): for pkg in self.vartree.dep_match(unvirt_pkg): ret.add(pkg) @@ -144,8 +144,8 @@ class PortageUtils: return ret def get_deps_for_package_building(self, pkg): - """ - returns buildtime dependencies of current package and + """ + returns buildtime dependencies of current package and all runtime dependencies of that buildtime dependencies """ buildtime_deps=self.get_dep(pkg, ["DEPEND"]) @@ -157,9 +157,9 @@ class PortageUtils: return ret def get_system_packages_list(self): - """ + """ returns all packages from system set. They are always implicit dependencies - + :returns: **list** of package names """ ret=[] @@ -172,11 +172,12 @@ class PortageUtils: class GentoolkitUtils: - """ - Interface with qfile and qlist utils. They are much faster than + """ + Interface with qfile and qlist utils. They are much faster than internals. """ + @staticmethod def getpackagesbyfiles(files): """ :param files: list of filenames @@ -190,14 +191,14 @@ class GentoolkitUtils: ret[f]="directory" else: listtocheck.append(f) - + try: proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) - + lines=out.decode("utf8").split("\n") #print lines line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") @@ -212,24 +213,25 @@ class GentoolkitUtils: except OSError as e: portage.util.writemsg("Error while launching qfile: %s\n" % e) - - + + return ret - + + @staticmethod def getfilesbypackages(packagenames): """ - + :param packagename: name of package :returns: **list** of files in package with name *packagename* """ ret=[] try: proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, - stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate() - + ret=out.decode("utf8").split("\n") if ret==['']: ret=[] @@ -237,34 +239,35 @@ class GentoolkitUtils: portage.util.writemsg("Error while launching qfile: %s\n" % e) return ret - - def get_all_packages_files(): + + @staticmethod + def get_all_packages_files(): """ Memory-hungry operation - + :returns: **set** of all files that belongs to package """ ret=[] try: proc=subprocess.Popen(['qlist']+['--all',"--obj"], - stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate() - + ret=out.decode("utf8").split("\n") except OSError as e: portage.util.writemsg("Error while launching qfile: %s\n" % e) return set(ret) - + class FilterProcGenerator: def __init__(self, pkgname, settings): portageutils=PortageUtils(settings=settings) deps_all=portageutils.get_deps_for_package_building(pkgname) deps_portage=portageutils.get_dep('portage',["RDEPEND"]) - + system_packages=portageutils.get_system_packages_list() allfiles=GentoolkitUtils.get_all_packages_files() @@ -272,8 +275,8 @@ class FilterProcGenerator: "a list of allowed files\n") - allowedpkgs=system_packages+list(deps_portage)+list(deps_all) - + allowedpkgs=system_packages+list(deps_portage)+list(deps_all) + allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) #for pkg in allowedpkgs: # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) @@ -283,14 +286,14 @@ class FilterProcGenerator: # manually add all python interpreters to this list allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) allowedfiles=set(allowedfiles) - + deniedfiles=allfiles-allowedfiles def filter_proc(eventname,filename,stage): if filename in deniedfiles: return False return True - + self.filter_proc=filter_proc def get_filter_proc(self): return self.filter_proc @@ -305,10 +308,10 @@ class EventsAnalyser: self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) - + self.system_packages=self.portageutils.get_system_packages_list() # All analyse work is here - + # get unique filenames filenames=set() for stage in events: @@ -319,7 +322,7 @@ class EventsAnalyser: filenames=list(filenames) file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) - # This part is completly unreadable. + # This part is completly unreadable. # It converting one complex struct(returned by getfsevents) to another complex # struct which good for generating output. # @@ -330,24 +333,24 @@ class EventsAnalyser: for stage in sorted(events): succ_events=events[stage][0] fail_events=events[stage][1] - + for filename in succ_events: if filename in file_to_package: package=file_to_package[filename] else: package="unknown" - + if not package in packagesinfo: packagesinfo[package]={} stageinfo=packagesinfo[package] if not stage in stageinfo: stageinfo[stage]={} - + filesinfo=stageinfo[stage] if not filename in filesinfo: filesinfo[filename]={"found":[],"notfound":[]} filesinfo[filename]["found"]=succ_events[filename] - + for filename in fail_events: if filename in file_to_package: package=file_to_package[filename] @@ -358,13 +361,13 @@ class EventsAnalyser: stageinfo=packagesinfo[package] if not stage in stageinfo: stageinfo[stage]={} - + filesinfo=stageinfo[stage] if not filename in filesinfo: filesinfo[filename]={"found":[],"notfound":[]} filesinfo[filename]["notfound"]=fail_events[filename] self.packagesinfo=packagesinfo - + def display(self): portage.util.writemsg( portage.output.colorize( @@ -373,12 +376,12 @@ class EventsAnalyser: stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} packagesinfo=self.packagesinfo - # print information grouped by package + # print information grouped by package for package in sorted(packagesinfo): # not showing special directory package if package=="directory": continue - + if package=="unknown": continue @@ -395,7 +398,7 @@ class EventsAnalyser: if len(stages)==0: continue - + filenames={} for stage in stages: for filename in packagesinfo[package][stage]: @@ -406,7 +409,7 @@ class EventsAnalyser: else: status, old_was_readed, old_was_writed=filenames[filename] filenames[filename]=[ - 'ok',old_was_readed | was_readed, old_was_writed | was_writed + 'ok',old_was_readed | was_readed, old_was_writed | was_writed ] if len(packagesinfo[package][stage][filename]["notfound"])!=0: was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] @@ -415,9 +418,9 @@ class EventsAnalyser: else: status, old_was_notfound, old_was_blocked=filenames[filename] filenames[filename]=[ - 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked + 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked ] - + if is_pkg_in_dep: portage.util.writemsg("[OK]") @@ -446,9 +449,9 @@ class EventsAnalyser: ('err',False,True):"blocked", ('err',True,True):"not found and blocked" } - + filescounter=0 - + for filename in filenames: event_info=tuple(filenames[filename]) portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) @@ -456,7 +459,7 @@ class EventsAnalyser: if filescounter>10: portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) break - # ... and one more check. Making sure that direct build time + # ... and one more check. Making sure that direct build time # dependencies were accessed #import pdb; pdb.set_trace() not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) @@ -465,7 +468,7 @@ class EventsAnalyser: portage.util.writemsg("Warning! Some build time dependencies " + \ "of packages were not accessed: " + \ " ".join(not_accessed_deps) + "\n") - + def is_package_useful(self,pkg,stages,files): """ some basic heuristics here to cut part of packages """ @@ -499,13 +502,13 @@ class EventsAnalyser: for f in files: if is_file_excluded(f): continue - - # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 + + # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): break else: return False # we get here if cycle ends not with break - + return True - -
\ No newline at end of file + + diff --git a/portage_with_autodep/pym/_emerge/EventsLogger.py b/portage_with_autodep/pym/_emerge/EventsLogger.py index 68b3c67..a08c533 100644 --- a/portage_with_autodep/pym/_emerge/EventsLogger.py +++ b/portage_with_autodep/pym/_emerge/EventsLogger.py @@ -14,17 +14,17 @@ from portage import os class EventsLogger(threading.Thread): def default_filter(eventname, filename, stage): return True - + def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): threading.Thread.__init__(self) # init the Thread - + self.alive=False - + self.main_thread=threading.currentThread() - + self.socket_dir=socket_dir self.filter_proc=filter_proc - + self.socket_name=None self.socket_logger=None @@ -33,16 +33,16 @@ class EventsLogger(threading.Thread): try: socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, prefix="log_socket_") - + socket_name = os.path.join(socket_dir_name, 'socket') except OSError as e: return - + self.socket_name=socket_name - + #print(self.socket_name) - + try: socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -65,21 +65,21 @@ class EventsLogger(threading.Thread): stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) except OSError as e: return - + def run(self): """ Starts the log server """ self.alive=True self.listen_thread=threading.currentThread() clients={} - + epoll=select.epoll() epoll.register(self.socket_logger.fileno(), select.EPOLLIN) while self.alive: try: sock_events = epoll.poll(3) - + for fileno, sock_event in sock_events: if fileno == self.socket_logger.fileno(): ret = self.socket_logger.accept() @@ -92,13 +92,13 @@ class EventsLogger(threading.Thread): elif sock_event & select.EPOLLIN: s=clients[fileno] record=s.recv(8192) - + if not record: # if connection was closed epoll.unregister(fileno) clients[fileno].close() del clients[fileno] continue - + #import pdb; pdb.set_trace() try: message=record.decode("utf8").split("\0") @@ -109,7 +109,7 @@ class EventsLogger(threading.Thread): # continue #print(message) - + try: if message[4]=="ASKING": if self.filter_proc(message[1],message[2],message[3]): @@ -123,7 +123,7 @@ class EventsLogger(threading.Thread): if not stage in self.events: self.events[stage]=[{},{}] - + hashofsucesses=self.events[stage][0] hashoffailures=self.events[stage][1] @@ -133,19 +133,19 @@ class EventsLogger(threading.Thread): if result=="OK": if not filename in hashofsucesses: hashofsucesses[filename]=[False,False] - + readed_or_writed=hashofsucesses[filename] - + if eventname=="read": readed_or_writed[0]=True elif eventname=="write": readed_or_writed[1]=True - + elif result[0:3]=="ERR" or result=="DENIED": if not filename in hashoffailures: hashoffailures[filename]=[False,False] notfound_or_blocked=hashoffailures[filename] - + if result=="ERR/2": notfound_or_blocked[0]=True elif result=="DENIED": @@ -153,28 +153,28 @@ class EventsLogger(threading.Thread): else: print("Error in logger module<->analyser protocol") - + except IndexError: print("IndexError while parsing %s" % record) except IOError as e: if e.errno!=4: # handling "Interrupted system call" errors raise - - # if main thread doesnt exists then exit + + # if main thread doesnt exists then exit if not self.main_thread.is_alive(): break epoll.unregister(self.socket_logger.fileno()) epoll.close() self.socket_logger.close() - + def stop(self): """ Stops the log server. Returns all events """ self.alive=False - + # Block the main thread until listener exists self.listen_thread.join() - + # We assume portage clears tmp folder, so no deleting a socket file # We assume that no new socket data will arrive after this moment return self.events diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.py b/portage_with_autodep/pym/_emerge/FakeVartree.py index d4dbe97..14be50c 100644 --- a/portage_with_autodep/pym/_emerge/FakeVartree.py +++ b/portage_with_autodep/pym/_emerge/FakeVartree.py @@ -1,6 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys import warnings @@ -10,11 +12,17 @@ from _emerge.Package import Package from _emerge.PackageVirtualDbapi import PackageVirtualDbapi from portage.const import VDB_PATH from portage.dbapi.vartree import vartree -from portage.repository.config import _gen_valid_repo +from portage.dep._slot_operator import find_built_slot_operator_atoms +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData, InvalidDependString from portage.update import grab_updates, parse_updates, update_dbentries +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode class FakeVardbapi(PackageVirtualDbapi): """ @@ -27,6 +35,9 @@ class FakeVardbapi(PackageVirtualDbapi): path =os.path.join(path, filename) return path +class _DynamicDepsNotApplicable(Exception): + pass + class FakeVartree(vartree): """This is implements an in-memory copy of a vartree instance that provides all the interfaces required for use by the depgraph. The vardb is locked @@ -39,9 +50,10 @@ class FakeVartree(vartree): is not a matching ebuild in the tree). Instances of this class are not populated until the sync() method is called.""" def __init__(self, root_config, pkg_cache=None, pkg_root_config=None, - dynamic_deps=True): + dynamic_deps=True, ignore_built_slot_operator_deps=False): self._root_config = root_config self._dynamic_deps = dynamic_deps + self._ignore_built_slot_operator_deps = ignore_built_slot_operator_deps if pkg_root_config is None: pkg_root_config = self._root_config self._pkg_root_config = pkg_root_config @@ -68,7 +80,7 @@ class FakeVartree(vartree): self.dbapi.aux_get = self._aux_get_wrapper self.dbapi.match = self._match_wrapper self._aux_get_history = set() - self._portdb_keys = ["EAPI", "DEPEND", "RDEPEND", "PDEPEND"] + self._portdb_keys = Package._dep_keys + ("EAPI", "KEYWORDS") self._portdb = portdb self._global_updates = None @@ -95,18 +107,30 @@ class FakeVartree(vartree): self._aux_get_wrapper(cpv, []) return matches - def _aux_get_wrapper(self, pkg, wants, myrepo=None): - if pkg in self._aux_get_history: - return self._aux_get(pkg, wants) - self._aux_get_history.add(pkg) - # We need to check the EAPI, and this also raises - # a KeyError to the caller if appropriate. - installed_eapi, repo = self._aux_get(pkg, ["EAPI", "repository"]) + def _aux_get_wrapper(self, cpv, wants, myrepo=None): + if cpv in self._aux_get_history: + return self._aux_get(cpv, wants) + self._aux_get_history.add(cpv) + + # This raises a KeyError to the caller if appropriate. + pkg = self.dbapi._cpv_map[cpv] + try: - # Use the live ebuild metadata if possible. - repo = _gen_valid_repo(repo) live_metadata = dict(zip(self._portdb_keys, - self._portdb.aux_get(pkg, self._portdb_keys, myrepo=repo))) + self._portdb.aux_get(cpv, self._portdb_keys, + myrepo=pkg.repo))) + except (KeyError, portage.exception.PortageException): + live_metadata = None + + self._apply_dynamic_deps(pkg, live_metadata) + + return self._aux_get(cpv, wants) + + def _apply_dynamic_deps(self, pkg, live_metadata): + + try: + if live_metadata is None: + raise _DynamicDepsNotApplicable() # Use the metadata from the installed instance if the EAPI # of either instance is unsupported, since if the installed # instance has an unsupported or corrupt EAPI then we don't @@ -116,16 +140,46 @@ class FakeVartree(vartree): # order to respect dep updates without revision bump or EAPI # bump, as in bug #368725. if not (portage.eapi_is_supported(live_metadata["EAPI"]) and \ - portage.eapi_is_supported(installed_eapi)): - raise KeyError(pkg) - self.dbapi.aux_update(pkg, live_metadata) - except (KeyError, portage.exception.PortageException): + portage.eapi_is_supported(pkg.eapi)): + raise _DynamicDepsNotApplicable() + + # preserve built slot/sub-slot := operator deps + built_slot_operator_atoms = None + if not self._ignore_built_slot_operator_deps and \ + _get_eapi_attrs(pkg.eapi).slot_operator: + try: + built_slot_operator_atoms = \ + find_built_slot_operator_atoms(pkg) + except InvalidDependString: + pass + + if built_slot_operator_atoms: + live_eapi_attrs = _get_eapi_attrs(live_metadata["EAPI"]) + if not live_eapi_attrs.slot_operator: + raise _DynamicDepsNotApplicable() + for k, v in built_slot_operator_atoms.items(): + live_metadata[k] += (" " + + " ".join(_unicode(atom) for atom in v)) + + self.dbapi.aux_update(pkg.cpv, live_metadata) + except _DynamicDepsNotApplicable: if self._global_updates is None: self._global_updates = \ grab_global_updates(self._portdb) + + # Bypass _aux_get_wrapper, since calling that + # here would trigger infinite recursion. + aux_keys = Package._dep_keys + self.dbapi._pkg_str_aux_keys + aux_dict = dict(zip(aux_keys, self._aux_get(pkg.cpv, aux_keys))) perform_global_updates( - pkg, self.dbapi, self._global_updates) - return self._aux_get(pkg, wants) + pkg.cpv, aux_dict, self.dbapi, self._global_updates) + + def dynamic_deps_preload(self, pkg, metadata): + if metadata is not None: + metadata = dict((k, metadata.get(k, '')) + for k in self._portdb_keys) + self._apply_dynamic_deps(pkg, metadata) + self._aux_get_history.add(pkg.cpv) def cpv_discard(self, pkg): """ @@ -223,12 +277,6 @@ class FakeVartree(vartree): root_config=self._pkg_root_config, type_name="installed") - try: - mycounter = long(pkg.metadata["COUNTER"]) - except ValueError: - mycounter = 0 - pkg.metadata["COUNTER"] = str(mycounter) - self._pkg_cache[pkg] = pkg return pkg @@ -257,12 +305,14 @@ def grab_global_updates(portdb): return retupdates -def perform_global_updates(mycpv, mydb, myupdates): - aux_keys = ["DEPEND", "RDEPEND", "PDEPEND", 'repository'] - aux_dict = dict(zip(aux_keys, mydb.aux_get(mycpv, aux_keys))) - repository = aux_dict.pop('repository') +def perform_global_updates(mycpv, aux_dict, mydb, myupdates): + try: + pkg = _pkg_str(mycpv, metadata=aux_dict, settings=mydb.settings) + except InvalidData: + return + aux_dict = dict((k, aux_dict[k]) for k in Package._dep_keys) try: - mycommands = myupdates[repository] + mycommands = myupdates[pkg.repo] except KeyError: try: mycommands = myupdates['DEFAULT'] @@ -272,6 +322,6 @@ def perform_global_updates(mycpv, mydb, myupdates): if not mycommands: return - updates = update_dbentries(mycommands, aux_dict) + updates = update_dbentries(mycommands, aux_dict, parent=pkg) if updates: mydb.aux_update(mycpv, updates) diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.pyo b/portage_with_autodep/pym/_emerge/FakeVartree.pyo Binary files differindex 8707391..740d33d 100644 --- a/portage_with_autodep/pym/_emerge/FakeVartree.pyo +++ b/portage_with_autodep/pym/_emerge/FakeVartree.pyo diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py index fcc4ab4..7468de5 100644 --- a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py @@ -1,6 +1,14 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import sys + +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + from portage import os from _emerge.AbstractPollTask import AbstractPollTask from portage.cache.mappings import slot_dict_class @@ -21,7 +29,18 @@ class FifoIpcDaemon(AbstractPollTask): self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - self._reg_id = self.scheduler.register( + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, + fcntl.fcntl(self._files.pipe_in, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch( self._files.pipe_in, self._registered_events, self._input_handler) @@ -32,11 +51,23 @@ class FifoIpcDaemon(AbstractPollTask): Re-open the input stream, in order to suppress POLLHUP events (bug #339976). """ - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) os.close(self._files.pipe_in) self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - self._reg_id = self.scheduler.register( + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, + fcntl.fcntl(self._files.pipe_in, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch( self._files.pipe_in, self._registered_events, self._input_handler) @@ -47,6 +78,8 @@ class FifoIpcDaemon(AbstractPollTask): if self.returncode is None: self.returncode = 1 self._unregister() + # notify exit listeners + self.wait() def _wait(self): if self.returncode is not None: @@ -67,7 +100,7 @@ class FifoIpcDaemon(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo Binary files differindex 6d7c4f9..15f04ac 100644 --- a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py index 5b9b221..9f6f09b 100644 --- a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py @@ -1,6 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import formatter import io import sys @@ -9,7 +11,6 @@ import time import portage from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode from portage.output import xtermTitle @@ -121,7 +122,8 @@ class JobStatusDisplay(object): term_codes = {} for k, capname in self._termcap_name_map.items(): - code = tigetstr(capname) + # Use _native_string for PyPy compat (bug #470258). + code = tigetstr(portage._native_string(capname)) if code is None: code = self._default_term_codes[capname] term_codes[k] = code @@ -233,10 +235,10 @@ class JobStatusDisplay(object): def _display_status(self): # Don't use len(self._completed_tasks) here since that also # can include uninstall tasks. - curval_str = str(self.curval) - maxval_str = str(self.maxval) - running_str = str(self.running) - failed_str = str(self.failed) + curval_str = "%s" % (self.curval,) + maxval_str = "%s" % (self.maxval,) + running_str = "%s" % (self.running,) + failed_str = "%s" % (self.failed,) load_avg_str = self._load_avg_str() color_output = io.StringIO() @@ -248,36 +250,36 @@ class JobStatusDisplay(object): f = formatter.AbstractFormatter(style_writer) number_style = "INFORM" - f.add_literal_data(_unicode_decode("Jobs: ")) + f.add_literal_data("Jobs: ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(curval_str)) + f.add_literal_data(curval_str) f.pop_style() - f.add_literal_data(_unicode_decode(" of ")) + f.add_literal_data(" of ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(maxval_str)) + f.add_literal_data(maxval_str) f.pop_style() - f.add_literal_data(_unicode_decode(" complete")) + f.add_literal_data(" complete") if self.running: - f.add_literal_data(_unicode_decode(", ")) + f.add_literal_data(", ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(running_str)) + f.add_literal_data(running_str) f.pop_style() - f.add_literal_data(_unicode_decode(" running")) + f.add_literal_data(" running") if self.failed: - f.add_literal_data(_unicode_decode(", ")) + f.add_literal_data(", ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(failed_str)) + f.add_literal_data(failed_str) f.pop_style() - f.add_literal_data(_unicode_decode(" failed")) + f.add_literal_data(" failed") padding = self._jobs_column_width - len(plain_output.getvalue()) if padding > 0: - f.add_literal_data(padding * _unicode_decode(" ")) + f.add_literal_data(padding * " ") - f.add_literal_data(_unicode_decode("Load avg: ")) - f.add_literal_data(_unicode_decode(load_avg_str)) + f.add_literal_data("Load avg: ") + f.add_literal_data(load_avg_str) # Truncate to fit width, to avoid making the terminal scroll if the # line overflows (happens when the load average is large). diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo Binary files differindex f79b2c2..551188d 100644 --- a/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.py b/portage_with_autodep/pym/_emerge/MergeListItem.py index 8086c68..172dfcc 100644 --- a/portage_with_autodep/pym/_emerge/MergeListItem.py +++ b/portage_with_autodep/pym/_emerge/MergeListItem.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -32,7 +32,7 @@ class MergeListItem(CompositeTask): if pkg.installed: # uninstall, executed by self.merge() self.returncode = os.EX_OK - self.wait() + self._async_wait() return args_set = self.args_set @@ -47,7 +47,9 @@ class MergeListItem(CompositeTask): action_desc = "Emerging" preposition = "for" + pkg_color = "PKG_MERGE" if pkg.type_name == "binary": + pkg_color = "PKG_BINARY_MERGE" action_desc += " binary" if build_opts.fetchonly: @@ -57,7 +59,7 @@ class MergeListItem(CompositeTask): (action_desc, colorize("MERGE_LIST_PROGRESS", str(pkg_count.curval)), colorize("MERGE_LIST_PROGRESS", str(pkg_count.maxval)), - colorize("GOOD", pkg.cpv)) + colorize(pkg_color, pkg.cpv)) portdb = pkg.root_config.trees["porttree"].dbapi portdir_repo_name = portdb.getRepositoryName(portdb.porttree_root) diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.pyo b/portage_with_autodep/pym/_emerge/MergeListItem.pyo Binary files differindex 168a227..b5295f0 100644 --- a/portage_with_autodep/pym/_emerge/MergeListItem.pyo +++ b/portage_with_autodep/pym/_emerge/MergeListItem.pyo diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.py b/portage_with_autodep/pym/_emerge/MetadataRegen.py index e82015f..d92b6a0 100644 --- a/portage_with_autodep/pym/_emerge/MetadataRegen.py +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.py @@ -1,18 +1,20 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage from portage import os from portage.dep import _repo_separator from _emerge.EbuildMetadataPhase import EbuildMetadataPhase -from _emerge.PollScheduler import PollScheduler +from portage.cache.cache_errors import CacheError +from portage.util._async.AsyncScheduler import AsyncScheduler -class MetadataRegen(PollScheduler): +class MetadataRegen(AsyncScheduler): def __init__(self, portdb, cp_iter=None, consumer=None, - max_jobs=None, max_load=None): - PollScheduler.__init__(self, main=True) + write_auxdb=True, **kwargs): + AsyncScheduler.__init__(self, **kwargs) self._portdb = portdb + self._write_auxdb = write_auxdb self._global_cleanse = False if cp_iter is None: cp_iter = self._iter_every_cp() @@ -22,34 +24,21 @@ class MetadataRegen(PollScheduler): self._cp_iter = cp_iter self._consumer = consumer - if max_jobs is None: - max_jobs = 1 - - self._max_jobs = max_jobs - self._max_load = max_load - self._valid_pkgs = set() self._cp_set = set() self._process_iter = self._iter_metadata_processes() - self.returncode = os.EX_OK - self._error_count = 0 self._running_tasks = set() - self._remaining_tasks = True - def _terminate_tasks(self): - for task in list(self._running_tasks): - task.cancel() + def _next_task(self): + return next(self._process_iter) def _iter_every_cp(self): - portage.writemsg_stdout("Listing available packages...\n") - every_cp = self._portdb.cp_all() - portage.writemsg_stdout("Regenerating cache entries...\n") - every_cp.sort(reverse=True) - try: - while not self._terminated_tasks: - yield every_cp.pop() - except IndexError: - pass + # List categories individually, in order to start yielding quicker, + # and in order to reduce latency in case of a signal interrupt. + cp_all = self._portdb.cp_all + for category in sorted(self._portdb.categories): + for cp in cp_all(categories=(category,)): + yield cp def _iter_metadata_processes(self): portdb = self._portdb @@ -57,8 +46,9 @@ class MetadataRegen(PollScheduler): cp_set = self._cp_set consumer = self._consumer + portage.writemsg_stdout("Regenerating cache entries...\n") for cp in self._cp_iter: - if self._terminated_tasks: + if self._terminated.is_set(): break cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) @@ -68,7 +58,7 @@ class MetadataRegen(PollScheduler): repo = portdb.repositories.get_repo_for_location(mytree) cpv_list = portdb.cp_list(cp, mytree=[repo.location]) for cpv in cpv_list: - if self._terminated_tasks: + if self._terminated.is_set(): break valid_pkgs.add(cpv) ebuild_path, repo_path = portdb.findname2(cpv, myrepo=repo.name) @@ -84,22 +74,21 @@ class MetadataRegen(PollScheduler): yield EbuildMetadataPhase(cpv=cpv, ebuild_hash=ebuild_hash, portdb=portdb, repo_path=repo_path, - settings=portdb.doebuild_settings) + settings=portdb.doebuild_settings, + write_auxdb=self._write_auxdb) - def _keep_scheduling(self): - return self._remaining_tasks and not self._terminated_tasks + def _wait(self): - def run(self): + AsyncScheduler._wait(self) portdb = self._portdb - from portage.cache.cache_errors import CacheError dead_nodes = {} - self._main_loop() - + self._termination_check() if self._terminated_tasks: - self.returncode = 1 - return + portdb.flush_cache() + self.returncode = self._cancelled_returncode + return self.returncode if self._global_cleanse: for mytree in portdb.porttrees: @@ -142,29 +131,12 @@ class MetadataRegen(PollScheduler): except (KeyError, CacheError): pass - def _schedule_tasks(self): - if self._terminated_tasks: - return - - while self._can_add_job(): - try: - metadata_process = next(self._process_iter) - except StopIteration: - self._remaining_tasks = False - return - - self._jobs += 1 - self._running_tasks.add(metadata_process) - metadata_process.scheduler = self.sched_iface - metadata_process.addExitListener(self._metadata_exit) - metadata_process.start() - - def _metadata_exit(self, metadata_process): - self._jobs -= 1 - self._running_tasks.discard(metadata_process) + portdb.flush_cache() + return self.returncode + + def _task_exit(self, metadata_process): + if metadata_process.returncode != os.EX_OK: - self.returncode = 1 - self._error_count += 1 self._valid_pkgs.discard(metadata_process.cpv) if not self._terminated_tasks: portage.writemsg("Error processing %s, continuing...\n" % \ @@ -179,5 +151,4 @@ class MetadataRegen(PollScheduler): metadata_process.ebuild_hash, metadata_process.eapi_supported) - self._schedule() - + AsyncScheduler._task_exit(self, metadata_process) diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.pyo b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo Binary files differindex 6c8788f..bcb940f 100644 --- a/portage_with_autodep/pym/_emerge/MetadataRegen.pyo +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py index afa44fb..bada79d 100644 --- a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -29,6 +29,10 @@ class MiscFunctionsProcess(AbstractEbuildProcess): AbstractEbuildProcess._start(self) def _spawn(self, args, **kwargs): + + if self._dummy_pipe_fd is not None: + self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + # Temporarily unset EBUILD_PHASE so that bashrc code doesn't # think this is a real phase. phase_backup = self.settings.pop("EBUILD_PHASE", None) @@ -37,3 +41,4 @@ class MiscFunctionsProcess(AbstractEbuildProcess): finally: if phase_backup is not None: self.settings["EBUILD_PHASE"] = phase_backup + self.settings.pop("PORTAGE_PIPE_FD", None) diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo Binary files differindex e3f5344..773256a 100644 --- a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Package.py b/portage_with_autodep/pym/_emerge/Package.py index c04fa1f..c795568 100644 --- a/portage_with_autodep/pym/_emerge/Package.py +++ b/portage_with_autodep/pym/_emerge/Package.py @@ -1,81 +1,98 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys from itertools import chain +import warnings + import portage from portage import _encodings, _unicode_decode, _unicode_encode from portage.cache.mappings import slot_dict_class from portage.const import EBUILD_PHASES from portage.dep import Atom, check_required_use, use_reduce, \ - paren_enclose, _slot_re, _slot_separator, _repo_separator + paren_enclose, _slot_separator, _repo_separator from portage.versions import _pkg_str, _unknown_repo -from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use +from portage.eapi import _get_eapi_attrs, eapi_has_use_aliases from portage.exception import InvalidDependString +from portage.localization import _ from _emerge.Task import Task if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class Package(Task): __hash__ = Task.__hash__ __slots__ = ("built", "cpv", "depth", - "installed", "metadata", "onlydeps", "operation", + "installed", "onlydeps", "operation", "root_config", "type_name", "category", "counter", "cp", "cpv_split", - "inherited", "invalid", "iuse", "masks", "mtime", - "pf", "root", "slot", "slot_atom", "version", "visible",) + \ - ("_raw_metadata", "_use",) + "inherited", "iuse", "mtime", + "pf", "root", "slot", "sub_slot", "slot_atom", "version") + \ + ("_invalid", "_masks", "_metadata", "_raw_metadata", "_use", + "_validated_atoms", "_visible") metadata_keys = [ "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI", - "INHERITED", "IUSE", "KEYWORDS", + "HDEPEND", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "PROPERTIES", "RESTRICT", "SLOT", "USE", "_mtime_", "DEFINED_PHASES", "REQUIRED_USE"] - _dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND',) + _dep_keys = ('DEPEND', 'HDEPEND', 'PDEPEND', 'RDEPEND') + _buildtime_keys = ('DEPEND', 'HDEPEND') + _runtime_keys = ('PDEPEND', 'RDEPEND') _use_conditional_misc_keys = ('LICENSE', 'PROPERTIES', 'RESTRICT') UNKNOWN_REPO = _unknown_repo def __init__(self, **kwargs): + metadata = _PackageMetadataWrapperBase(kwargs.pop('metadata')) Task.__init__(self, **kwargs) # the SlotObject constructor assigns self.root_config from keyword args # and is an instance of a '_emerge.RootConfig.RootConfig class self.root = self.root_config.root - self._raw_metadata = _PackageMetadataWrapperBase(self.metadata) - self.metadata = _PackageMetadataWrapper(self, self._raw_metadata) + self._raw_metadata = metadata + self._metadata = _PackageMetadataWrapper(self, metadata) if not self.built: - self.metadata['CHOST'] = self.root_config.settings.get('CHOST', '') - slot = self.slot - if _slot_re.match(slot) is None: + self._metadata['CHOST'] = self.root_config.settings.get('CHOST', '') + eapi_attrs = _get_eapi_attrs(self.eapi) + self.cpv = _pkg_str(self.cpv, metadata=self._metadata, + settings=self.root_config.settings) + if hasattr(self.cpv, 'slot_invalid'): self._invalid_metadata('SLOT.invalid', - "SLOT: invalid value: '%s'" % slot) - # Avoid an InvalidAtom exception when creating slot_atom. - # This package instance will be masked due to empty SLOT. - slot = '0' - self.cpv = _pkg_str(self.cpv, slot=slot, - repo=self.metadata.get('repository', '')) + "SLOT: invalid value: '%s'" % self._metadata["SLOT"]) + self.cpv_split = self.cpv.cpv_split + self.category, self.pf = portage.catsplit(self.cpv) self.cp = self.cpv.cp + self.version = self.cpv.version + self.slot = self.cpv.slot + self.sub_slot = self.cpv.sub_slot + self.slot_atom = Atom("%s%s%s" % (self.cp, _slot_separator, self.slot)) # sync metadata with validated repo (may be UNKNOWN_REPO) - self.metadata['repository'] = self.cpv.repo + self._metadata['repository'] = self.cpv.repo + + if eapi_attrs.iuse_effective: + implicit_match = self.root_config.settings._iuse_effective_match + else: + implicit_match = self.root_config.settings._iuse_implicit_match + usealiases = self.root_config.settings._use_manager.getUseAliases(self) + self.iuse = self._iuse(self, self._metadata["IUSE"].split(), implicit_match, + usealiases, self.eapi) + if (self.iuse.enabled or self.iuse.disabled) and \ - not eapi_has_iuse_defaults(self.metadata["EAPI"]): + not eapi_attrs.iuse_defaults: if not self.installed: self._invalid_metadata('EAPI.incompatible', "IUSE contains defaults, but EAPI doesn't allow them") - self.slot_atom = portage.dep.Atom("%s%s%s" % (self.cp, _slot_separator, slot)) - self.category, self.pf = portage.catsplit(self.cpv) - self.cpv_split = self.cpv.cpv_split - self.version = self.cpv.version if self.inherited is None: self.inherited = frozenset() - self._validate_deps() - self.masks = self._masks() - self.visible = self._visible(self.masks) if self.operation is None: if self.onlydeps or self.installed: self.operation = "nomerge" @@ -89,6 +106,74 @@ class Package(Task): type_name=self.type_name) self._hash_value = hash(self._hash_key) + @property + def eapi(self): + return self._metadata["EAPI"] + + @property + def build_time(self): + if not self.built: + raise AttributeError('build_time') + try: + return long(self._metadata['BUILD_TIME']) + except (KeyError, ValueError): + return 0 + + @property + def defined_phases(self): + return self._metadata.defined_phases + + @property + def properties(self): + return self._metadata.properties + + @property + def restrict(self): + return self._metadata.restrict + + @property + def metadata(self): + warnings.warn("_emerge.Package.Package.metadata is deprecated", + DeprecationWarning, stacklevel=3) + return self._metadata + + # These are calculated on-demand, so that they are calculated + # after FakeVartree applies its metadata tweaks. + @property + def invalid(self): + if self._invalid is None: + self._validate_deps() + if self._invalid is None: + self._invalid = False + return self._invalid + + @property + def masks(self): + if self._masks is None: + self._masks = self._eval_masks() + return self._masks + + @property + def visible(self): + if self._visible is None: + self._visible = self._eval_visiblity(self.masks) + return self._visible + + @property + def validated_atoms(self): + """ + Returns *all* validated atoms from the deps, regardless + of USE conditionals, with USE conditionals inside + atoms left unevaluated. + """ + if self._validated_atoms is None: + self._validate_deps() + return self._validated_atoms + + @property + def stable(self): + return self.cpv.stable + @classmethod def _gen_hash_key(cls, cpv=None, installed=None, onlydeps=None, operation=None, repo_name=None, root_config=None, @@ -123,15 +208,15 @@ class Package(Task): # So overwrite the repo_key with type_name. repo_key = type_name - return (type_name, root, cpv, operation, repo_key) + return (type_name, root, _unicode(cpv), operation, repo_key) def _validate_deps(self): """ Validate deps. This does not trigger USE calculation since that is expensive for ebuilds and therefore we want to avoid doing - in unnecessarily (like for masked packages). + it unnecessarily (like for masked packages). """ - eapi = self.metadata['EAPI'] + eapi = self.eapi dep_eapi = eapi dep_valid_flag = self.iuse.is_valid_flag if self.installed: @@ -142,28 +227,44 @@ class Package(Task): dep_eapi = None dep_valid_flag = None + validated_atoms = [] for k in self._dep_keys: - v = self.metadata.get(k) + v = self._metadata.get(k) if not v: continue try: - use_reduce(v, eapi=dep_eapi, matchall=True, - is_valid_flag=dep_valid_flag, token_class=Atom) + atoms = use_reduce(v, eapi=dep_eapi, + matchall=True, is_valid_flag=dep_valid_flag, + token_class=Atom, flat=True) except InvalidDependString as e: self._metadata_exception(k, e) + else: + validated_atoms.extend(atoms) + if not self.built: + for atom in atoms: + if not isinstance(atom, Atom): + continue + if atom.slot_operator_built: + e = InvalidDependString( + _("Improper context for slot-operator " + "\"built\" atom syntax: %s") % + (atom.unevaluated_atom,)) + self._metadata_exception(k, e) + + self._validated_atoms = tuple(set(atom for atom in + validated_atoms if isinstance(atom, Atom))) k = 'PROVIDE' - v = self.metadata.get(k) + v = self._metadata.get(k) if v: try: use_reduce(v, eapi=dep_eapi, matchall=True, is_valid_flag=dep_valid_flag, token_class=Atom) except InvalidDependString as e: - self._invalid_metadata("PROVIDE.syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata("PROVIDE.syntax", "%s: %s" % (k, e)) for k in self._use_conditional_misc_keys: - v = self.metadata.get(k) + v = self._metadata.get(k) if not v: continue try: @@ -173,24 +274,20 @@ class Package(Task): self._metadata_exception(k, e) k = 'REQUIRED_USE' - v = self.metadata.get(k) - if v: - if not eapi_has_required_use(eapi): + v = self._metadata.get(k) + if v and not self.built: + if not _get_eapi_attrs(eapi).required_use: self._invalid_metadata('EAPI.incompatible', "REQUIRED_USE set, but EAPI='%s' doesn't allow it" % eapi) else: try: check_required_use(v, (), - self.iuse.is_valid_flag) + self.iuse.is_valid_flag, eapi=eapi) except InvalidDependString as e: - # Force unicode format string for python-2.x safety, - # ensuring that PortageException.__unicode__() is used - # when necessary. - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata(k + ".syntax", "%s: %s" % (k, e)) k = 'SRC_URI' - v = self.metadata.get(k) + v = self._metadata.get(k) if v: try: use_reduce(v, is_src_uri=True, eapi=eapi, matchall=True, @@ -205,43 +302,52 @@ class Package(Task): onlydeps=self.onlydeps, operation=self.operation, root_config=self.root_config, type_name=self.type_name) - def _masks(self): + def _eval_masks(self): masks = {} settings = self.root_config.settings - if self.invalid is not None: + if self.invalid is not False: masks['invalid'] = self.invalid - if not settings._accept_chost(self.cpv, self.metadata): - masks['CHOST'] = self.metadata['CHOST'] + if not settings._accept_chost(self.cpv, self._metadata): + masks['CHOST'] = self._metadata['CHOST'] - eapi = self.metadata["EAPI"] + eapi = self.eapi if not portage.eapi_is_supported(eapi): masks['EAPI.unsupported'] = eapi if portage._eapi_is_deprecated(eapi): masks['EAPI.deprecated'] = eapi missing_keywords = settings._getMissingKeywords( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_keywords: masks['KEYWORDS'] = missing_keywords try: missing_properties = settings._getMissingProperties( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_properties: masks['PROPERTIES'] = missing_properties except InvalidDependString: # already recorded as 'invalid' pass - mask_atom = settings._getMaskAtom(self.cpv, self.metadata) + try: + missing_restricts = settings._getMissingRestrict( + self.cpv, self._metadata) + if missing_restricts: + masks['RESTRICT'] = missing_restricts + except InvalidDependString: + # already recorded as 'invalid' + pass + + mask_atom = settings._getMaskAtom(self.cpv, self._metadata) if mask_atom is not None: masks['package.mask'] = mask_atom try: missing_licenses = settings._getMissingLicenses( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_licenses: masks['LICENSE'] = missing_licenses except InvalidDependString: @@ -249,13 +355,13 @@ class Package(Task): pass if not masks: - masks = None + masks = False return masks - def _visible(self, masks): + def _eval_visiblity(self, masks): - if masks is not None: + if masks is not False: if 'EAPI.unsupported' in masks: return False @@ -267,7 +373,8 @@ class Package(Task): 'CHOST' in masks or \ 'EAPI.deprecated' in masks or \ 'KEYWORDS' in masks or \ - 'PROPERTIES' in masks): + 'PROPERTIES' in masks or \ + 'RESTRICT' in masks): return False if 'package.mask' in masks or \ @@ -280,7 +387,7 @@ class Package(Task): """returns None, 'missing', or 'unstable'.""" missing = self.root_config.settings._getRawMissingKeywords( - self.cpv, self.metadata) + self.cpv, self._metadata) if not missing: return None @@ -301,17 +408,22 @@ class Package(Task): """returns a bool if the cpv is in the list of expanded pmaskdict[cp] available ebuilds""" pmask = self.root_config.settings._getRawMaskAtom( - self.cpv, self.metadata) + self.cpv, self._metadata) return pmask is not None def _metadata_exception(self, k, e): + if k.endswith('DEPEND'): + qacat = 'dependency.syntax' + else: + qacat = k + ".syntax" + # For unicode safety with python-2.x we need to avoid # using the string format operator with a non-unicode # format string, since that will result in the # PortageException.__str__() method being invoked, # followed by unsafe decoding that may result in a - # UnicodeDecodeError. Therefore, use _unicode_decode() + # UnicodeDecodeError. Therefore, use unicode_literals # to ensure that format strings are unicode, so that # PortageException.__unicode__() is used when necessary # in python-2.x. @@ -323,27 +435,25 @@ class Package(Task): continue categorized_error = True self._invalid_metadata(error.category, - _unicode_decode("%s: %s") % (k, error)) + "%s: %s" % (k, error)) if not categorized_error: - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata(qacat,"%s: %s" % (k, e)) else: # For installed packages, show the path of the file # containing the invalid metadata, since the user may # want to fix the deps by hand. vardb = self.root_config.trees['vartree'].dbapi path = vardb.getpath(self.cpv, filename=k) - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s in '%s'") % (k, e, path)) + self._invalid_metadata(qacat, "%s: %s in '%s'" % (k, e, path)) def _invalid_metadata(self, msg_type, msg): - if self.invalid is None: - self.invalid = {} - msgs = self.invalid.get(msg_type) + if self._invalid is None: + self._invalid = {} + msgs = self._invalid.get(msg_type) if msgs is None: msgs = [] - self.invalid[msg_type] = msgs + self._invalid[msg_type] = msgs msgs.append(msg) def __str__(self): @@ -389,13 +499,16 @@ class Package(Task): # Share identical frozenset instances when available. _frozensets = {} - def __init__(self, pkg, use_str): + def __init__(self, pkg, enabled_flags): self._pkg = pkg self._expand = None self._expand_hidden = None self._force = None self._mask = None - self.enabled = frozenset(use_str.split()) + if eapi_has_use_aliases(pkg.eapi): + for enabled_flag in enabled_flags: + enabled_flags.extend(pkg.iuse.alias_mapping.get(enabled_flag, [])) + self.enabled = frozenset(enabled_flags) if pkg.built: # Use IUSE to validate USE settings for built packages, # in case the package manager that built this package @@ -445,7 +558,7 @@ class Package(Task): @property def repo(self): - return self.metadata['repository'] + return self._metadata['repository'] @property def repo_priority(self): @@ -457,7 +570,7 @@ class Package(Task): @property def use(self): if self._use is None: - self.metadata._init_use() + self._init_use() return self._use def _get_pkgsettings(self): @@ -466,28 +579,81 @@ class Package(Task): pkgsettings.setcpv(self) return pkgsettings + def _init_use(self): + if self.built: + # Use IUSE to validate USE settings for built packages, + # in case the package manager that built this package + # failed to do that for some reason (or in case of + # data corruption). The enabled flags must be consistent + # with implicit IUSE, in order to avoid potential + # inconsistencies in USE dep matching (see bug #453400). + use_str = self._metadata['USE'] + is_valid_flag = self.iuse.is_valid_flag + enabled_flags = [x for x in use_str.split() if is_valid_flag(x)] + use_str = " ".join(enabled_flags) + self._use = self._use_class( + self, enabled_flags) + else: + try: + use_str = _PackageMetadataWrapperBase.__getitem__( + self._metadata, 'USE') + except KeyError: + use_str = None + calculated_use = False + if not use_str: + use_str = self._get_pkgsettings()["PORTAGE_USE"] + calculated_use = True + self._use = self._use_class( + self, use_str.split()) + # Initialize these now, since USE access has just triggered + # setcpv, and we want to cache the result of the force/mask + # calculations that were done. + if calculated_use: + self._use._init_force_mask() + + _PackageMetadataWrapperBase.__setitem__( + self._metadata, 'USE', use_str) + + return use_str + class _iuse(object): - __slots__ = ("__weakref__", "all", "enabled", "disabled", - "tokens") + ("_iuse_implicit_match",) + __slots__ = ("__weakref__", "_iuse_implicit_match", "_pkg", "alias_mapping", + "all", "all_aliases", "enabled", "disabled", "tokens") - def __init__(self, tokens, iuse_implicit_match): + def __init__(self, pkg, tokens, iuse_implicit_match, aliases, eapi): + self._pkg = pkg self.tokens = tuple(tokens) self._iuse_implicit_match = iuse_implicit_match enabled = [] disabled = [] other = [] + enabled_aliases = [] + disabled_aliases = [] + other_aliases = [] + aliases_supported = eapi_has_use_aliases(eapi) + self.alias_mapping = {} for x in tokens: prefix = x[:1] if prefix == "+": enabled.append(x[1:]) + if aliases_supported: + self.alias_mapping[x[1:]] = aliases.get(x[1:], []) + enabled_aliases.extend(self.alias_mapping[x[1:]]) elif prefix == "-": disabled.append(x[1:]) + if aliases_supported: + self.alias_mapping[x[1:]] = aliases.get(x[1:], []) + disabled_aliases.extend(self.alias_mapping[x[1:]]) else: other.append(x) - self.enabled = frozenset(enabled) - self.disabled = frozenset(disabled) + if aliases_supported: + self.alias_mapping[x] = aliases.get(x, []) + other_aliases.extend(self.alias_mapping[x]) + self.enabled = frozenset(chain(enabled, enabled_aliases)) + self.disabled = frozenset(chain(disabled, disabled_aliases)) self.all = frozenset(chain(enabled, disabled, other)) + self.all_aliases = frozenset(chain(enabled_aliases, disabled_aliases, other_aliases)) def is_valid_flag(self, flags): """ @@ -498,7 +664,7 @@ class Package(Task): flags = [flags] for flag in flags: - if not flag in self.all and \ + if not flag in self.all and not flag in self.all_aliases and \ not self._iuse_implicit_match(flag): return False return True @@ -511,11 +677,28 @@ class Package(Task): flags = [flags] missing_iuse = [] for flag in flags: - if not flag in self.all and \ + if not flag in self.all and not flag in self.all_aliases and \ not self._iuse_implicit_match(flag): missing_iuse.append(flag) return missing_iuse + def get_real_flag(self, flag): + """ + Returns the flag's name within the scope of this package + (accounting for aliases), or None if the flag is unknown. + """ + if flag in self.all: + return flag + elif flag in self.all_aliases: + for k, v in self.alias_mapping.items(): + if flag in v: + return k + + if self._iuse_implicit_match(flag): + return flag + + return None + def __len__(self): return 4 @@ -568,7 +751,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): __slots__ = ("_pkg",) _wrapped_keys = frozenset( - ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) + ["COUNTER", "INHERITED", "USE", "_mtime_"]) _use_conditional_keys = frozenset( ['LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',]) @@ -581,31 +764,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): self.update(metadata) - def _init_use(self): - if self._pkg.built: - use_str = self['USE'] - self._pkg._use = self._pkg._use_class( - self._pkg, use_str) - else: - try: - use_str = _PackageMetadataWrapperBase.__getitem__(self, 'USE') - except KeyError: - use_str = None - calculated_use = False - if not use_str: - use_str = self._pkg._get_pkgsettings()["PORTAGE_USE"] - calculated_use = True - _PackageMetadataWrapperBase.__setitem__(self, 'USE', use_str) - self._pkg._use = self._pkg._use_class( - self._pkg, use_str) - # Initialize these now, since USE access has just triggered - # setcpv, and we want to cache the result of the force/mask - # calculations that were done. - if calculated_use: - self._pkg._use._init_force_mask() - - return use_str - def __getitem__(self, k): v = _PackageMetadataWrapperBase.__getitem__(self, k) if k in self._use_conditional_keys: @@ -623,7 +781,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): elif k == 'USE' and not self._pkg.built: if not v: # This is lazy because it's expensive. - v = self._init_use() + v = self._pkg._init_use() return v @@ -637,13 +795,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): v = frozenset(v.split()) self._pkg.inherited = v - def _set_iuse(self, k, v): - self._pkg.iuse = self._pkg._iuse( - v.split(), self._pkg.root_config.settings._iuse_implicit_match) - - def _set_slot(self, k, v): - self._pkg.slot = v - def _set_counter(self, k, v): if isinstance(v, basestring): try: diff --git a/portage_with_autodep/pym/_emerge/Package.pyo b/portage_with_autodep/pym/_emerge/Package.pyo Binary files differindex 3d37317..6bec2fe 100644 --- a/portage_with_autodep/pym/_emerge/Package.pyo +++ b/portage_with_autodep/pym/_emerge/Package.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageArg.pyo b/portage_with_autodep/pym/_emerge/PackageArg.pyo Binary files differindex c50e145..2367aee 100644 --- a/portage_with_autodep/pym/_emerge/PackageArg.pyo +++ b/portage_with_autodep/pym/_emerge/PackageArg.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.py b/portage_with_autodep/pym/_emerge/PackageMerge.py index eed34e9..ef298ca 100644 --- a/portage_with_autodep/pym/_emerge/PackageMerge.py +++ b/portage_with_autodep/pym/_emerge/PackageMerge.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.CompositeTask import CompositeTask @@ -11,6 +11,9 @@ class PackageMerge(CompositeTask): self.scheduler = self.merge.scheduler pkg = self.merge.pkg pkg_count = self.merge.pkg_count + pkg_color = "PKG_MERGE" + if pkg.type_name == "binary": + pkg_color = "PKG_BINARY_MERGE" if pkg.installed: action_desc = "Uninstalling" @@ -26,7 +29,7 @@ class PackageMerge(CompositeTask): msg = "%s %s%s" % \ (action_desc, counter_str, - colorize("GOOD", pkg.cpv)) + colorize(pkg_color, pkg.cpv)) if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.pyo b/portage_with_autodep/pym/_emerge/PackageMerge.pyo Binary files differindex 6403d1b..171b38c 100644 --- a/portage_with_autodep/pym/_emerge/PackageMerge.pyo +++ b/portage_with_autodep/pym/_emerge/PackageMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageUninstall.py b/portage_with_autodep/pym/_emerge/PackageUninstall.py index eb6a947..16c2f74 100644 --- a/portage_with_autodep/pym/_emerge/PackageUninstall.py +++ b/portage_with_autodep/pym/_emerge/PackageUninstall.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import logging @@ -33,7 +33,7 @@ class PackageUninstall(CompositeTask): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK - self.wait() + self._async_wait() return self.settings.setcpv(self.pkg) @@ -67,7 +67,7 @@ class PackageUninstall(CompositeTask): if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval - self.wait() + self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), diff --git a/portage_with_autodep/pym/_emerge/PackageUninstall.pyo b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo Binary files differindex 847c749..e70bec2 100644 --- a/portage_with_autodep/pym/_emerge/PackageUninstall.pyo +++ b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py index 0f7be44..56a5576 100644 --- a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py @@ -140,10 +140,10 @@ class PackageVirtualDbapi(dbapi): self._clear_cache() def aux_get(self, cpv, wants, myrepo=None): - metadata = self._cpv_map[cpv].metadata + metadata = self._cpv_map[cpv]._metadata return [metadata.get(x, "") for x in wants] def aux_update(self, cpv, values): - self._cpv_map[cpv].metadata.update(values) + self._cpv_map[cpv]._metadata.update(values) self._clear_cache() diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo Binary files differindex a1a850f..f9c6641 100644 --- a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo diff --git a/portage_with_autodep/pym/_emerge/PipeReader.py b/portage_with_autodep/pym/_emerge/PipeReader.py index 90febdf..a8392c3 100644 --- a/portage_with_autodep/pym/_emerge/PipeReader.py +++ b/portage_with_autodep/pym/_emerge/PipeReader.py @@ -1,9 +1,11 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import fcntl +import sys + from portage import os from _emerge.AbstractPollTask import AbstractPollTask -import fcntl class PipeReader(AbstractPollTask): @@ -27,18 +29,28 @@ class PipeReader(AbstractPollTask): output_handler = self._output_handler for f in self.input_files.values(): - fcntl.fcntl(f.fileno(), fcntl.F_SETFL, - fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_ids.add(self.scheduler.register(f.fileno(), + fd = isinstance(f, int) and f or f.fileno() + fcntl.fcntl(fd, fcntl.F_SETFL, + fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(fd, fcntl.F_SETFD, + fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_ids.add(self.scheduler.io_add_watch(fd, self._registered_events, output_handler)) self._registered = True - def isAlive(self): - return self._registered - def _cancel(self): + self._unregister() if self.returncode is None: - self.returncode = 1 + self.returncode = self._cancelled_returncode def _wait(self): if self.returncode is not None: @@ -102,11 +114,14 @@ class PipeReader(AbstractPollTask): if self._reg_ids is not None: for reg_id in self._reg_ids: - self.scheduler.unregister(reg_id) + self.scheduler.source_remove(reg_id) self._reg_ids = None if self.input_files is not None: for f in self.input_files.values(): - f.close() + if isinstance(f, int): + os.close(f) + else: + f.close() self.input_files = None diff --git a/portage_with_autodep/pym/_emerge/PipeReader.pyo b/portage_with_autodep/pym/_emerge/PipeReader.pyo Binary files differindex 2f53e7d..1ad40c2 100644 --- a/portage_with_autodep/pym/_emerge/PipeReader.pyo +++ b/portage_with_autodep/pym/_emerge/PipeReader.pyo diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.py b/portage_with_autodep/pym/_emerge/PollScheduler.py index 965dc20..b118ac1 100644 --- a/portage_with_autodep/pym/_emerge/PollScheduler.py +++ b/portage_with_autodep/pym/_emerge/PollScheduler.py @@ -1,18 +1,13 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import gzip -import errno - try: import threading except ImportError: import dummy_threading as threading -from portage import _encodings -from portage import _unicode_encode -from portage.util import writemsg_level -from portage.util.SlotObject import SlotObject +import portage +from portage.util._async.SchedulerInterface import SchedulerInterface from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop @@ -20,14 +15,10 @@ from _emerge.getloadavg import getloadavg class PollScheduler(object): - class _sched_iface_class(SlotObject): - __slots__ = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", - "IO_PRI", "child_watch_add", - "idle_add", "io_add_watch", "iteration", - "output", "register", "run", - "source_remove", "timeout_add", "unregister") + # max time between loadavg checks (milliseconds) + _loadavg_latency = None - def __init__(self, main=False): + def __init__(self, main=False, event_loop=None): """ @param main: If True then use global_event_loop(), otherwise use a local EventLoop instance (default is False, for safe use in @@ -38,29 +29,20 @@ class PollScheduler(object): self._terminated_tasks = False self._max_jobs = 1 self._max_load = None - self._jobs = 0 self._scheduling = False self._background = False - if main: + if event_loop is not None: + self._event_loop = event_loop + elif main: self._event_loop = global_event_loop() else: - self._event_loop = EventLoop(main=False) - self.sched_iface = self._sched_iface_class( - IO_ERR=self._event_loop.IO_ERR, - IO_HUP=self._event_loop.IO_HUP, - IO_IN=self._event_loop.IO_IN, - IO_NVAL=self._event_loop.IO_NVAL, - IO_OUT=self._event_loop.IO_OUT, - IO_PRI=self._event_loop.IO_PRI, - child_watch_add=self._event_loop.child_watch_add, - idle_add=self._event_loop.idle_add, - io_add_watch=self._event_loop.io_add_watch, - iteration=self._event_loop.iteration, - output=self._task_output, - register=self._event_loop.io_add_watch, - source_remove=self._event_loop.source_remove, - timeout_add=self._event_loop.timeout_add, - unregister=self._event_loop.source_remove) + self._event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) + self._sched_iface = SchedulerInterface(self._event_loop, + is_background=self._is_background) + + def _is_background(self): + return self._background def terminate(self): """ @@ -135,41 +117,23 @@ class PollScheduler(object): Calls _schedule_tasks() and automatically returns early from any recursive calls to this method that the _schedule_tasks() call might trigger. This makes _schedule() safe to call from - inside exit listeners. + inside exit listeners. This method always returns True, so that + it may be scheduled continuously via EventLoop.timeout_add(). """ if self._scheduling: - return False + return True self._scheduling = True try: self._schedule_tasks() finally: self._scheduling = False - - def _main_loop(self): - term_check_id = self.sched_iface.idle_add(self._termination_check) - try: - # Populate initial event sources. We only need to do - # this once here, since it can be called during the - # loop from within event handlers. - self._schedule() - - # Loop while there are jobs to be scheduled. - while self._keep_scheduling(): - self.sched_iface.iteration() - - # Clean shutdown of previously scheduled jobs. In the - # case of termination, this allows for basic cleanup - # such as flushing of buffered output to logs. - while self._is_work_scheduled(): - self.sched_iface.iteration() - finally: - self.sched_iface.source_remove(term_check_id) + return True def _is_work_scheduled(self): return bool(self._running_job_count()) def _running_job_count(self): - return self._jobs + raise NotImplementedError(self) def _can_add_job(self): if self._terminated_tasks: @@ -194,47 +158,3 @@ class PollScheduler(object): return False return True - - def _task_output(self, msg, log_path=None, background=None, - level=0, noiselevel=-1): - """ - Output msg to stdout if not self._background. If log_path - is not None then append msg to the log (appends with - compression if the filename extension of log_path - corresponds to a supported compression type). - """ - - if background is None: - # If the task does not have a local background value - # (like for parallel-fetch), then use the global value. - background = self._background - - msg_shown = False - if not background: - writemsg_level(msg, level=level, noiselevel=noiselevel) - msg_shown = True - - if log_path is not None: - try: - f = open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='ab') - f_real = f - except IOError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - if not msg_shown: - writemsg_level(msg, level=level, noiselevel=noiselevel) - else: - - if log_path.endswith('.gz'): - # NOTE: The empty filename argument prevents us from - # triggering a bug in python3 which causes GzipFile - # to raise AttributeError if fileobj.name is bytes - # instead of unicode. - f = gzip.GzipFile(filename='', mode='ab', fileobj=f) - - f.write(_unicode_encode(msg)) - f.close() - if f_real is not f: - f_real.close() diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.pyo b/portage_with_autodep/pym/_emerge/PollScheduler.pyo Binary files differindex b7e52be..d714e01 100644 --- a/portage_with_autodep/pym/_emerge/PollScheduler.pyo +++ b/portage_with_autodep/pym/_emerge/PollScheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/ProgressHandler.pyo b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo Binary files differindex 83e2f7f..fd0c5bb 100644 --- a/portage_with_autodep/pym/_emerge/ProgressHandler.pyo +++ b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.py b/portage_with_autodep/pym/_emerge/QueueScheduler.py deleted file mode 100644 index 206087c..0000000 --- a/portage_with_autodep/pym/_emerge/QueueScheduler.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 1999-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from _emerge.PollScheduler import PollScheduler - -class QueueScheduler(PollScheduler): - - """ - Add instances of SequentialTaskQueue and then call run(). The - run() method returns when no tasks remain. - """ - - def __init__(self, main=True, max_jobs=None, max_load=None): - PollScheduler.__init__(self, main=main) - - if max_jobs is None: - max_jobs = 1 - - self._max_jobs = max_jobs - self._max_load = max_load - - self._queues = [] - self._schedule_listeners = [] - - def add(self, q): - self._queues.append(q) - - def remove(self, q): - self._queues.remove(q) - - def clear(self): - for q in self._queues: - q.clear() - - def run(self, timeout=None): - - timeout_callback = None - if timeout is not None: - def timeout_callback(): - timeout_callback.timed_out = True - return False - timeout_callback.timed_out = False - timeout_callback.timeout_id = self.sched_iface.timeout_add( - timeout, timeout_callback) - - term_check_id = self.sched_iface.idle_add(self._termination_check) - try: - while not (timeout_callback is not None and - timeout_callback.timed_out): - # We don't have any callbacks to trigger _schedule(), - # so we have to call it explicitly here. - self._schedule() - if self._keep_scheduling(): - self.sched_iface.iteration() - else: - break - - while self._is_work_scheduled() and \ - not (timeout_callback is not None and - timeout_callback.timed_out): - self.sched_iface.iteration() - finally: - self.sched_iface.source_remove(term_check_id) - if timeout_callback is not None: - self.sched_iface.unregister(timeout_callback.timeout_id) - - def _schedule_tasks(self): - """ - @rtype: bool - @return: True if there may be remaining tasks to schedule, - False otherwise. - """ - if self._terminated_tasks: - return - - while self._can_add_job(): - n = self._max_jobs - self._running_job_count() - if n < 1: - break - - if not self._start_next_job(n): - return - - def _keep_scheduling(self): - return not self._terminated_tasks and any(self._queues) - - def _running_job_count(self): - job_count = 0 - for q in self._queues: - job_count += len(q.running_tasks) - self._jobs = job_count - return job_count - - def _start_next_job(self, n=1): - started_count = 0 - for q in self._queues: - initial_job_count = len(q.running_tasks) - q.schedule() - final_job_count = len(q.running_tasks) - if final_job_count > initial_job_count: - started_count += (final_job_count - initial_job_count) - if started_count >= n: - break - return started_count - diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.pyo b/portage_with_autodep/pym/_emerge/QueueScheduler.pyo Binary files differdeleted file mode 100644 index 88de3ea..0000000 --- a/portage_with_autodep/pym/_emerge/QueueScheduler.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/_emerge/RootConfig.py b/portage_with_autodep/pym/_emerge/RootConfig.py index bb0d768..3648d01 100644 --- a/portage_with_autodep/pym/_emerge/RootConfig.py +++ b/portage_with_autodep/pym/_emerge/RootConfig.py @@ -1,10 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 class RootConfig(object): """This is used internally by depgraph to track information about a particular $ROOT.""" - __slots__ = ("root", "setconfig", "sets", "settings", "trees") + __slots__ = ("mtimedb", "root", "setconfig", "sets", "settings", "trees") pkg_tree_map = { "ebuild" : "porttree", @@ -31,4 +31,11 @@ class RootConfig(object): Shallow copy all attributes from another instance. """ for k in self.__slots__: - setattr(self, k, getattr(other, k)) + try: + setattr(self, k, getattr(other, k)) + except AttributeError: + # mtimedb is currently not a required attribute + try: + delattr(self, k) + except AttributeError: + pass diff --git a/portage_with_autodep/pym/_emerge/RootConfig.pyo b/portage_with_autodep/pym/_emerge/RootConfig.pyo Binary files differindex fad3022..d071898 100644 --- a/portage_with_autodep/pym/_emerge/RootConfig.pyo +++ b/portage_with_autodep/pym/_emerge/RootConfig.pyo diff --git a/portage_with_autodep/pym/_emerge/Scheduler.py b/portage_with_autodep/pym/_emerge/Scheduler.py index 30a7e10..d663e97 100644 --- a/portage_with_autodep/pym/_emerge/Scheduler.py +++ b/portage_with_autodep/pym/_emerge/Scheduler.py @@ -1,7 +1,7 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals from collections import deque import gc @@ -18,7 +18,7 @@ import zlib import portage from portage import os from portage import _encodings -from portage import _unicode_decode, _unicode_encode +from portage import _unicode_encode from portage.cache.mappings import slot_dict_class from portage.elog.messages import eerror from portage.localization import _ @@ -28,6 +28,8 @@ from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.SlotObject import SlotObject +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.digestgen import digestgen from portage.package.ebuild.doebuild import (_check_temp_dir, @@ -50,6 +52,7 @@ from _emerge.EbuildFetcher import EbuildFetcher from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog from _emerge.FakeVartree import FakeVartree +from _emerge.getloadavg import getloadavg from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo from _emerge.JobStatusDisplay import JobStatusDisplay @@ -64,6 +67,9 @@ if sys.hexversion >= 0x3000000: class Scheduler(PollScheduler): + # max time between loadavg checks (milliseconds) + _loadavg_latency = 30000 + # max time between display status updates (milliseconds) _max_display_latency = 3000 @@ -79,7 +85,7 @@ class Scheduler(PollScheduler): _opts_no_self_update = frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) - class _iface_class(PollScheduler._sched_iface_class): + class _iface_class(SchedulerInterface): __slots__ = ("fetch", "scheduleSetup", "scheduleUnpack") @@ -135,8 +141,7 @@ class Scheduler(PollScheduler): portage.exception.PortageException.__init__(self, value) def __init__(self, settings, trees, mtimedb, myopts, - spinner, mergelist=None, favorites=None, graph_config=None, - uninstall_only=False): + spinner, mergelist=None, favorites=None, graph_config=None): PollScheduler.__init__(self, main=True) if mergelist is not None: @@ -152,7 +157,6 @@ class Scheduler(PollScheduler): self._spinner = spinner self._mtimedb = mtimedb self._favorites = favorites - self._uninstall_only = uninstall_only self._args_set = InternalPackageSet(favorites, allow_repo=True) self._build_opts = self._build_opts_class() @@ -217,14 +221,15 @@ class Scheduler(PollScheduler): fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( + self._event_loop, + is_background=self._is_background, fetch=fetch_iface, scheduleSetup=self._schedule_setup, - scheduleUnpack=self._schedule_unpack, - **dict((k, getattr(self.sched_iface, k)) - for k in self.sched_iface.__slots__)) + scheduleUnpack=self._schedule_unpack) self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] + self._jobs = 0 self._running_tasks = {} self._completed_tasks = set() @@ -243,10 +248,15 @@ class Scheduler(PollScheduler): # The load average takes some time to respond when new # jobs are added, so we need to limit the rate of adding # new jobs. - self._job_delay_max = 10 - self._job_delay_factor = 1.0 - self._job_delay_exp = 1.5 + self._job_delay_max = 5 self._previous_job_start_time = None + self._job_delay_timeout_id = None + + # The load average takes some time to respond when after + # a SIGSTOP/SIGCONT cycle, so delay scheduling for some + # time after SIGCONT is received. + self._sigcont_delay = 5 + self._sigcont_time = None # This is used to memoize the _choose_pkg() result when # no packages can be chosen until one of the existing @@ -300,15 +310,10 @@ class Scheduler(PollScheduler): if not portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [x]): continue - if self._running_portage is None or \ - self._running_portage.cpv != x.cpv or \ - '9999' in x.cpv or \ - 'git' in x.inherited or \ - 'git-2' in x.inherited: - rval = _check_temp_dir(self.settings) - if rval != os.EX_OK: - return rval - _prepare_self_update(self.settings) + rval = _check_temp_dir(self.settings) + if rval != os.EX_OK: + return rval + _prepare_self_update(self.settings) break return os.EX_OK @@ -328,12 +333,13 @@ class Scheduler(PollScheduler): self._set_graph_config(graph_config) self._blocker_db = {} dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_operator_deps = self.myopts.get( + "--ignore-built-slot-operator-deps", "n") == "y" for root in self.trees: - if self._uninstall_only: - continue if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps, + ignore_built_slot_operator_deps=ignore_built_slot_operator_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] @@ -410,7 +416,7 @@ class Scheduler(PollScheduler): if not (isinstance(task, Package) and \ task.operation == "merge"): continue - if 'interactive' in task.metadata.properties: + if 'interactive' in task.properties: interactive_tasks.append(task) return interactive_tasks @@ -655,10 +661,11 @@ class Scheduler(PollScheduler): if value and value.strip(): continue msg = _("%(var)s is not set... " - "Are you missing the '%(configroot)setc/make.profile' symlink? " + "Are you missing the '%(configroot)s%(profile_path)s' symlink? " "Is the symlink correct? " "Is your portage tree complete?") % \ - {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"]} + {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"], + "profile_path": portage.const.PROFILE_PATH} out = portage.output.EOutput() for line in textwrap.wrap(msg, 70): @@ -718,7 +725,6 @@ class Scheduler(PollScheduler): return if self._parallel_fetch: - self._status_msg("Starting parallel fetch") prefetchers = self._prefetchers @@ -771,10 +777,10 @@ class Scheduler(PollScheduler): failures = 0 - # Use a local PollScheduler instance here, since we don't + # Use a local EventLoop instance here, since we don't # want tasks here to trigger the usual Scheduler callbacks # that handle job scheduling and status display. - sched_iface = PollScheduler().sched_iface + sched_iface = SchedulerInterface(EventLoop(main=False)) for x in self._mergelist: if not isinstance(x, Package): @@ -783,10 +789,10 @@ class Scheduler(PollScheduler): if x.operation == "uninstall": continue - if x.metadata["EAPI"] in ("0", "1", "2", "3"): + if x.eapi in ("0", "1", "2", "3"): continue - if "pretend" not in x.metadata.defined_phases: + if "pretend" not in x.defined_phases: continue out_str =">>> Running pre-merge checks for " + colorize("INFORM", x.cpv) + "\n" @@ -805,7 +811,7 @@ class Scheduler(PollScheduler): build_dir_path = os.path.join( os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", x.category, x.pf) - existing_buildir = os.path.isdir(build_dir_path) + existing_builddir = os.path.isdir(build_dir_path) settings["PORTAGE_BUILDDIR"] = build_dir_path build_dir = EbuildBuildDir(scheduler=sched_iface, settings=settings) @@ -816,7 +822,7 @@ class Scheduler(PollScheduler): # Clean up the existing build dir, in case pkg_pretend # checks for available space (bug #390711). - if existing_buildir: + if existing_builddir: if x.built: tree = "bintree" infloc = os.path.join(build_dir_path, "build-info") @@ -905,13 +911,18 @@ class Scheduler(PollScheduler): failures += 1 portage.elog.elog_process(x.cpv, settings) finally: - if current_task is not None and current_task.isAlive(): - current_task.cancel() - current_task.wait() - clean_phase = EbuildPhase(background=False, - phase='clean', scheduler=sched_iface, settings=settings) - clean_phase.start() - clean_phase.wait() + + if current_task is not None: + if current_task.isAlive(): + current_task.cancel() + current_task.wait() + if current_task.returncode == os.EX_OK: + clean_phase = EbuildPhase(background=False, + phase='clean', scheduler=sched_iface, + settings=settings) + clean_phase.start() + clean_phase.wait() + build_dir.unlock() if failures: @@ -1001,6 +1012,8 @@ class Scheduler(PollScheduler): earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler) earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler) + earlier_sigcont_handler = \ + signal.signal(signal.SIGCONT, self._sigcont_handler) try: rval = self._merge() @@ -1014,6 +1027,10 @@ class Scheduler(PollScheduler): signal.signal(signal.SIGTERM, earlier_sigterm_handler) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) + if earlier_sigcont_handler is not None: + signal.signal(signal.SIGCONT, earlier_sigcont_handler) + else: + signal.signal(signal.SIGCONT, signal.SIG_DFL) if received_signal: sys.exit(received_signal[0]) @@ -1060,7 +1077,8 @@ class Scheduler(PollScheduler): printer = portage.output.EOutput() background = self._background failure_log_shown = False - if background and len(self._failed_pkgs_all) == 1: + if background and len(self._failed_pkgs_all) == 1 and \ + self.myopts.get('--quiet-fail', 'n') != 'y': # If only one package failed then just show it's # whole log for easy viewing. failed_pkg = self._failed_pkgs_all[-1] @@ -1139,9 +1157,9 @@ class Scheduler(PollScheduler): printer.eerror(line) printer.eerror("") for failed_pkg in self._failed_pkgs_all: - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that Package.__unicode__() is called in python2. - msg = _unicode_decode(" %s") % (failed_pkg.pkg,) + msg = " %s" % (failed_pkg.pkg,) log_path = self._locate_failure_log(failed_pkg) if log_path is not None: msg += ", Log file:" @@ -1338,6 +1356,38 @@ class Scheduler(PollScheduler): blocker_db = self._blocker_db[pkg.root] blocker_db.discardBlocker(pkg) + def _main_loop(self): + term_check_id = self._event_loop.idle_add(self._termination_check) + loadavg_check_id = None + if self._max_load is not None and \ + self._loadavg_latency is not None and \ + (self._max_jobs is True or self._max_jobs > 1): + # We have to schedule periodically, in case the load + # average has changed since the last call. + loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_latency, self._schedule) + + try: + # Populate initial event sources. Unless we're scheduling + # based on load average, we only need to do this once + # here, since it can be called during the loop from within + # event handlers. + self._schedule() + + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self._event_loop.iteration() + + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self._event_loop.iteration() + finally: + self._event_loop.source_remove(term_check_id) + if loadavg_check_id is not None: + self._event_loop.source_remove(loadavg_check_id) + def _merge(self): if self._opts_no_background.intersection(self.myopts): @@ -1348,8 +1398,10 @@ class Scheduler(PollScheduler): failed_pkgs = self._failed_pkgs portage.locks._quiet = self._background portage.elog.add_listener(self._elog_listener) - display_timeout_id = self.sched_iface.timeout_add( - self._max_display_latency, self._status_display.display) + display_timeout_id = None + if self._status_display._isatty and not self._status_display.quiet: + display_timeout_id = self._event_loop.timeout_add( + self._max_display_latency, self._status_display.display) rval = os.EX_OK try: @@ -1358,7 +1410,8 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.locks._quiet = False portage.elog.remove_listener(self._elog_listener) - self.sched_iface.source_remove(display_timeout_id) + if display_timeout_id is not None: + self._event_loop.source_remove(display_timeout_id) if failed_pkgs: rval = failed_pkgs[-1].returncode @@ -1490,12 +1543,15 @@ class Scheduler(PollScheduler): self._config_pool[settings['EROOT']].append(settings) def _keep_scheduling(self): - return bool(not self._terminated_tasks and self._pkg_queue and \ + return bool(not self._terminated.is_set() and self._pkg_queue and \ not (self._failed_pkgs and not self._build_opts.fetchonly)) def _is_work_scheduled(self): return bool(self._running_tasks) + def _running_job_count(self): + return self._jobs + def _schedule_tasks(self): while True: @@ -1536,6 +1592,9 @@ class Scheduler(PollScheduler): not self._task_queues.merge)): break + def _sigcont_handler(self, signum, frame): + self._sigcont_time = time.time() + def _job_delay(self): """ @rtype: bool @@ -1546,14 +1605,53 @@ class Scheduler(PollScheduler): current_time = time.time() - delay = self._job_delay_factor * self._jobs ** self._job_delay_exp + if self._sigcont_time is not None: + + elapsed_seconds = current_time - self._sigcont_time + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds > 0 and \ + elapsed_seconds < self._sigcont_delay: + + if self._job_delay_timeout_id is not None: + self._event_loop.source_remove( + self._job_delay_timeout_id) + + self._job_delay_timeout_id = self._event_loop.timeout_add( + 1000 * (self._sigcont_delay - elapsed_seconds), + self._schedule_once) + return True + + # Only set this to None after the delay has expired, + # since this method may be called again before the + # delay has expired. + self._sigcont_time = None + + try: + avg1, avg5, avg15 = getloadavg() + except OSError: + return False + + delay = self._job_delay_max * avg1 / self._max_load if delay > self._job_delay_max: delay = self._job_delay_max - if (current_time - self._previous_job_start_time) < delay: + elapsed_seconds = current_time - self._previous_job_start_time + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds > 0 and elapsed_seconds < delay: + + if self._job_delay_timeout_id is not None: + self._event_loop.source_remove( + self._job_delay_timeout_id) + + self._job_delay_timeout_id = self._event_loop.timeout_add( + 1000 * (delay - elapsed_seconds), self._schedule_once) return True return False + def _schedule_once(self): + self._schedule() + return False + def _schedule_tasks_imp(self): """ @rtype: bool @@ -1735,7 +1833,7 @@ class Scheduler(PollScheduler): # scope e = exc mydepgraph = e.depgraph - dropped_tasks = set() + dropped_tasks = {} if e is not None: def unsatisfied_resume_dep_msg(): @@ -1772,11 +1870,7 @@ class Scheduler(PollScheduler): return False if success and self._show_list(): - mylist = mydepgraph.altlist() - if mylist: - if "--tree" in self.myopts: - mylist.reverse() - mydepgraph.display(mylist, favorites=self._favorites) + mydepgraph.display(mydepgraph.altlist(), favorites=self._favorites) if not success: self._post_mod_echo_msgs.append(mydepgraph.display_problems) @@ -1785,7 +1879,7 @@ class Scheduler(PollScheduler): self._init_graph(mydepgraph.schedulerGraph()) msg_width = 75 - for task in dropped_tasks: + for task, atoms in dropped_tasks.items(): if not (isinstance(task, Package) and task.operation == "merge"): continue pkg = task @@ -1793,7 +1887,10 @@ class Scheduler(PollScheduler): " %s" % (pkg.cpv,) if pkg.root_config.settings["ROOT"] != "/": msg += " for %s" % (pkg.root,) - msg += " dropped due to unsatisfied dependency." + if not atoms: + msg += " dropped because it is masked or unavailable" + else: + msg += " dropped because it requires %s" % ", ".join(atoms) for line in textwrap.wrap(msg, msg_width): eerror(line, phase="other", key=pkg.cpv) settings = self.pkgsettings[pkg.root] @@ -1838,11 +1935,21 @@ class Scheduler(PollScheduler): root_config = pkg.root_config world_set = root_config.sets["selected"] world_locked = False - if hasattr(world_set, "lock"): - world_set.lock() - world_locked = True + atom = None + + if pkg.operation != "uninstall": + # Do this before acquiring the lock, since it queries the + # portdbapi which can call the global event loop, triggering + # a concurrent call to this method or something else that + # needs an exclusive (non-reentrant) lock on the world file. + atom = create_world_atom(pkg, args_set, root_config) try: + + if hasattr(world_set, "lock"): + world_set.lock() + world_locked = True + if hasattr(world_set, "load"): world_set.load() # maybe it's changed on disk @@ -1854,8 +1961,7 @@ class Scheduler(PollScheduler): for s in pkg.root_config.setconfig.active: world_set.remove(SETPREFIX+s) else: - atom = create_world_atom(pkg, args_set, root_config) - if atom: + if atom is not None: if hasattr(world_set, "add"): self._status_msg(('Recording %s in "world" ' + \ 'favorites file...') % atom) diff --git a/portage_with_autodep/pym/_emerge/Scheduler.pyo b/portage_with_autodep/pym/_emerge/Scheduler.pyo Binary files differindex 5555703..e61506d 100644 --- a/portage_with_autodep/pym/_emerge/Scheduler.pyo +++ b/portage_with_autodep/pym/_emerge/Scheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo Binary files differindex 3ab65c9..abf4b59 100644 --- a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo +++ b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo diff --git a/portage_with_autodep/pym/_emerge/SetArg.py b/portage_with_autodep/pym/_emerge/SetArg.py index 94cf0a6..5c82975 100644 --- a/portage_with_autodep/pym/_emerge/SetArg.py +++ b/portage_with_autodep/pym/_emerge/SetArg.py @@ -1,9 +1,12 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.DependencyArg import DependencyArg from portage._sets import SETPREFIX class SetArg(DependencyArg): + + __slots__ = ('name', 'pset') + def __init__(self, pset=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.pset = pset diff --git a/portage_with_autodep/pym/_emerge/SetArg.pyo b/portage_with_autodep/pym/_emerge/SetArg.pyo Binary files differindex 5a3d9d9..bc86247 100644 --- a/portage_with_autodep/pym/_emerge/SetArg.pyo +++ b/portage_with_autodep/pym/_emerge/SetArg.pyo diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.py b/portage_with_autodep/pym/_emerge/SpawnProcess.py index 9fbc964..15d3dc5 100644 --- a/portage_with_autodep/pym/_emerge/SpawnProcess.py +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.py @@ -1,17 +1,23 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 2008-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.SubProcess import SubProcess +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + +import errno +import logging +import signal import sys -from portage.cache.mappings import slot_dict_class + +from _emerge.SubProcess import SubProcess import portage -from portage import _encodings -from portage import _unicode_encode from portage import os from portage.const import BASH_BINARY -import fcntl -import errno -import gzip +from portage.util import writemsg_level +from portage.util._async.PipeLogger import PipeLogger class SpawnProcess(SubProcess): @@ -23,31 +29,27 @@ class SpawnProcess(SubProcess): _spawn_kwarg_names = ("env", "opt_name", "fd_pipes", "uid", "gid", "groups", "umask", "logfile", - "path_lookup", "pre_exec") + "path_lookup", "pre_exec", "close_fds", "cgroup", + "unshare_ipc", "unshare_net") __slots__ = ("args",) + \ - _spawn_kwarg_names + ("_log_file_real", "_selinux_type",) - - _file_names = ("log", "process", "stdout") - _files_dict = slot_dict_class(_file_names, prefix="") + _spawn_kwarg_names + ("_pipe_logger", "_selinux_type",) def _start(self): if self.fd_pipes is None: self.fd_pipes = {} + else: + self.fd_pipes = self.fd_pipes.copy() fd_pipes = self.fd_pipes - self._files = self._files_dict() - files = self._files - master_fd, slave_fd = self._pipe(fd_pipes) - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - files.process = master_fd - logfile = None - if self._can_log(slave_fd): - logfile = self.logfile + can_log = self._can_log(slave_fd) + if can_log: + log_file_path = self.logfile + else: + log_file_path = None null_input = None if not self.background or 0 in fd_pipes: @@ -62,48 +64,34 @@ class SpawnProcess(SubProcess): null_input = os.open('/dev/null', os.O_RDWR) fd_pipes[0] = null_input - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) + fd_pipes.setdefault(0, portage._get_stdin().fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() + if fd in stdout_filenos: + sys.__stdout__.flush() + sys.__stderr__.flush() + break - if logfile is not None: + fd_pipes_orig = fd_pipes.copy() - fd_pipes_orig = fd_pipes.copy() + if log_file_path is not None or self.background: fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd - files.log = open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), mode='ab') - if logfile.endswith('.gz'): - self._log_file_real = files.log - files.log = gzip.GzipFile(filename='', mode='ab', - fileobj=files.log) - - portage.util.apply_secpass_permissions(logfile, - uid=portage.portage_uid, gid=portage.portage_gid, - mode=0o660) - - if not self.background: - files.stdout = os.dup(fd_pipes_orig[1]) - - output_handler = self._output_handler - else: - - # Create a dummy pipe so the scheduler can monitor - # the process from inside a poll() loop. - fd_pipes[self._dummy_pipe_fd] = slave_fd - if self.background: - fd_pipes[1] = slave_fd - fd_pipes[2] = slave_fd - output_handler = self._dummy_handler + # Create a dummy pipe that PipeLogger uses to efficiently + # monitor for process exit by listening for the EOF event. + # Re-use of the allocated fd number for the key in fd_pipes + # guarantees that the keys will not collide for similarly + # allocated pipes which are used by callers such as + # FileDigester and MergeProcess. See the _setup_pipes + # docstring for more benefits of this allocation approach. + self._dummy_pipe_fd = slave_fd + fd_pipes[slave_fd] = slave_fd kwargs = {} for k in self._spawn_kwarg_names: @@ -115,10 +103,6 @@ class SpawnProcess(SubProcess): kwargs["returnpid"] = True kwargs.pop("logfile", None) - self._reg_id = self.scheduler.register(files.process, - self._registered_events, output_handler) - self._registered = True - retval = self._spawn(self.args, **kwargs) os.close(slave_fd) @@ -129,11 +113,32 @@ class SpawnProcess(SubProcess): # spawn failed self._unregister() self._set_returncode((self.pid, retval)) - self.wait() + self._async_wait() return self.pid = retval[0] - portage.process.spawned_pids.remove(self.pid) + + stdout_fd = None + if can_log and not self.background: + stdout_fd = os.dup(fd_pipes_orig[1]) + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(stdout_fd, fcntl.F_SETFD, + fcntl.fcntl(stdout_fd, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._pipe_logger = PipeLogger(background=self.background, + scheduler=self.scheduler, input_fd=master_fd, + log_file_path=log_file_path, + stdout_fd=stdout_fd) + self._pipe_logger.addExitListener(self._pipe_logger_exit) + self._pipe_logger.start() + self._registered = True def _can_log(self, slave_fd): return True @@ -157,92 +162,56 @@ class SpawnProcess(SubProcess): return spawn_func(args, **kwargs) - def _output_handler(self, fd, event): - - files = self._files - while True: - buf = self._read_buf(fd, event) - - if buf is None: - # not a POLLIN event, EAGAIN, etc... - break - - if not buf: - # EOF - self._unregister() - self.wait() - break - - else: - if not self.background: - write_successful = False - failures = 0 - while True: - try: - if not write_successful: - os.write(files.stdout, buf) - write_successful = True - break - except OSError as e: - if e.errno != errno.EAGAIN: - raise - del e - failures += 1 - if failures > 50: - # Avoid a potentially infinite loop. In - # most cases, the failure count is zero - # and it's unlikely to exceed 1. - raise - - # This means that a subprocess has put an inherited - # stdio file descriptor (typically stdin) into - # O_NONBLOCK mode. This is not acceptable (see bug - # #264435), so revert it. We need to use a loop - # here since there's a race condition due to - # parallel processes being able to change the - # flags on the inherited file descriptor. - # TODO: When possible, avoid having child processes - # inherit stdio file descriptors from portage - # (maybe it can't be avoided with - # PROPERTIES=interactive). - fcntl.fcntl(files.stdout, fcntl.F_SETFL, - fcntl.fcntl(files.stdout, - fcntl.F_GETFL) ^ os.O_NONBLOCK) - - files.log.write(buf) - files.log.flush() - - self._unregister_if_appropriate(event) - - return True - - def _dummy_handler(self, fd, event): - """ - This method is mainly interested in detecting EOF, since - the only purpose of the pipe is to allow the scheduler to - monitor the process from inside a poll() loop. - """ - - while True: - buf = self._read_buf(fd, event) - - if buf is None: - # not a POLLIN event, EAGAIN, etc... - break - - if not buf: - # EOF - self._unregister() - self.wait() - break - - self._unregister_if_appropriate(event) - - return True - - def _unregister(self): - super(SpawnProcess, self)._unregister() - if self._log_file_real is not None: - # Avoid "ResourceWarning: unclosed file" since python 3.2. - self._log_file_real.close() - self._log_file_real = None + def _pipe_logger_exit(self, pipe_logger): + self._pipe_logger = None + self._unregister() + self.wait() + + def _waitpid_loop(self): + SubProcess._waitpid_loop(self) + + pipe_logger = self._pipe_logger + if pipe_logger is not None: + self._pipe_logger = None + pipe_logger.removeExitListener(self._pipe_logger_exit) + pipe_logger.cancel() + pipe_logger.wait() + + def _set_returncode(self, wait_retval): + SubProcess._set_returncode(self, wait_retval) + + if self.cgroup: + def get_pids(cgroup): + try: + with open(os.path.join(cgroup, 'cgroup.procs'), 'r') as f: + return [int(p) for p in f.read().split()] + except OSError: + # cgroup removed already? + return [] + + def kill_all(pids, sig): + for p in pids: + try: + os.kill(p, sig) + except OSError as e: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (p,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: + raise + + # step 1: kill all orphans + pids = get_pids(self.cgroup) + if pids: + kill_all(pids, signal.SIGKILL) + + # step 2: remove the cgroup + try: + os.rmdir(self.cgroup) + except OSError: + # it may be removed already, or busy + # we can't do anything good about it + pass diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.pyo b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo Binary files differindex 7a6142e..b32e588 100644 --- a/portage_with_autodep/pym/_emerge/SpawnProcess.pyo +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/SubProcess.py b/portage_with_autodep/pym/_emerge/SubProcess.py index 76b313f..13d9382 100644 --- a/portage_with_autodep/pym/_emerge/SubProcess.py +++ b/portage_with_autodep/pym/_emerge/SubProcess.py @@ -1,7 +1,10 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import logging + from portage import os +from portage.util import writemsg_level from _emerge.AbstractPollTask import AbstractPollTask import signal import errno @@ -9,12 +12,7 @@ import errno class SubProcess(AbstractPollTask): __slots__ = ("pid",) + \ - ("_files", "_reg_id") - - # A file descriptor is required for the scheduler to monitor changes from - # inside a poll() loop. When logging is not enabled, create a pipe just to - # serve this purpose alone. - _dummy_pipe_fd = 9 + ("_dummy_pipe_fd", "_files", "_reg_id") # This is how much time we allow for waitpid to succeed after # we've sent a kill signal to our subprocess. @@ -50,7 +48,13 @@ class SubProcess(AbstractPollTask): try: os.kill(self.pid, signal.SIGTERM) except OSError as e: - if e.errno != errno.ESRCH: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (self.pid,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: raise def isAlive(self): @@ -69,7 +73,13 @@ class SubProcess(AbstractPollTask): try: os.kill(self.pid, signal.SIGKILL) except OSError as e: - if e.errno != errno.ESRCH: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (self.pid,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: raise del e self._wait_loop(timeout=self._cancel_timeout) @@ -116,7 +126,7 @@ class SubProcess(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/SubProcess.pyo b/portage_with_autodep/pym/_emerge/SubProcess.pyo Binary files differindex 26e13e1..0fb99fb 100644 --- a/portage_with_autodep/pym/_emerge/SubProcess.pyo +++ b/portage_with_autodep/pym/_emerge/SubProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Task.py b/portage_with_autodep/pym/_emerge/Task.py index 40f5066..250d458 100644 --- a/portage_with_autodep/pym/_emerge/Task.py +++ b/portage_with_autodep/pym/_emerge/Task.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.util.SlotObject import SlotObject @@ -41,3 +41,10 @@ class Task(SlotObject): strings. """ return "(%s)" % ", ".join(("'%s'" % x for x in self._hash_key)) + + def __repr__(self): + if self._hash_key is None: + # triggered by python-trace + return SlotObject.__repr__(self) + return "<%s (%s)>" % (self.__class__.__name__, + ", ".join(("'%s'" % x for x in self._hash_key))) diff --git a/portage_with_autodep/pym/_emerge/Task.pyo b/portage_with_autodep/pym/_emerge/Task.pyo Binary files differindex 2958cb1..eb71dac 100644 --- a/portage_with_autodep/pym/_emerge/Task.pyo +++ b/portage_with_autodep/pym/_emerge/Task.pyo diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.py b/portage_with_autodep/pym/_emerge/TaskScheduler.py deleted file mode 100644 index 583bfe3..0000000 --- a/portage_with_autodep/pym/_emerge/TaskScheduler.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 1999-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from _emerge.QueueScheduler import QueueScheduler -from _emerge.SequentialTaskQueue import SequentialTaskQueue - -class TaskScheduler(object): - - """ - A simple way to handle scheduling of AsynchrousTask instances. Simply - add tasks and call run(). The run() method returns when no tasks remain. - """ - - def __init__(self, main=True, max_jobs=None, max_load=None): - self._queue = SequentialTaskQueue(max_jobs=max_jobs) - self._scheduler = QueueScheduler(main=main, - max_jobs=max_jobs, max_load=max_load) - self.sched_iface = self._scheduler.sched_iface - self.run = self._scheduler.run - self.clear = self._scheduler.clear - self.wait = self._queue.wait - self._scheduler.add(self._queue) - - def add(self, task): - self._queue.add(task) - diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.pyo b/portage_with_autodep/pym/_emerge/TaskScheduler.pyo Binary files differdeleted file mode 100644 index 8b84de7..0000000 --- a/portage_with_autodep/pym/_emerge/TaskScheduler.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/_emerge/TaskSequence.pyo b/portage_with_autodep/pym/_emerge/TaskSequence.pyo Binary files differindex b98196e..0257937 100644 --- a/portage_with_autodep/pym/_emerge/TaskSequence.pyo +++ b/portage_with_autodep/pym/_emerge/TaskSequence.pyo diff --git a/portage_with_autodep/pym/_emerge/UninstallFailure.pyo b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo Binary files differindex 9f1c88b..50fba3d 100644 --- a/portage_with_autodep/pym/_emerge/UninstallFailure.pyo +++ b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo diff --git a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py index 4316600..ec44a67 100644 --- a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py +++ b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractDepPriority import AbstractDepPriority @@ -7,15 +7,16 @@ class UnmergeDepPriority(AbstractDepPriority): """ Combination of properties Priority Category - runtime 0 HARD - runtime_post -1 HARD - buildtime -2 SOFT - (none of the above) -2 SOFT + runtime_slot_op 0 HARD + runtime -1 HARD + runtime_post -2 HARD + buildtime -3 SOFT + (none of the above) -3 SOFT """ MAX = 0 - SOFT = -2 - MIN = -2 + SOFT = -3 + MIN = -3 def __init__(self, **kwargs): AbstractDepPriority.__init__(self, **kwargs) @@ -23,17 +24,21 @@ class UnmergeDepPriority(AbstractDepPriority): self.optional = True def __int__(self): - if self.runtime: + if self.runtime_slot_op: return 0 - if self.runtime_post: + if self.runtime: return -1 - if self.buildtime: + if self.runtime_post: return -2 - return -2 + if self.buildtime: + return -3 + return -3 def __str__(self): if self.ignored: return "ignored" + if self.runtime_slot_op: + return "hard slot op" myvalue = self.__int__() if myvalue > self.SOFT: return "hard" diff --git a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo Binary files differindex b163ed7..8f0e204 100644 --- a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/UseFlagDisplay.py b/portage_with_autodep/pym/_emerge/UseFlagDisplay.py index 3daca19..f460474 100644 --- a/portage_with_autodep/pym/_emerge/UseFlagDisplay.py +++ b/portage_with_autodep/pym/_emerge/UseFlagDisplay.py @@ -1,10 +1,12 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + from itertools import chain import sys -from portage import _encodings, _unicode_decode, _unicode_encode +from portage import _encodings, _unicode_encode from portage.output import red from portage.util import cmp_sort_key from portage.output import blue @@ -114,9 +116,9 @@ def pkg_use_display(pkg, opts, modified_use=None): flags.sort(key=UseFlagDisplay.sort_combined) else: flags.sort(key=UseFlagDisplay.sort_separated) - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that UseFlagDisplay.__unicode__() is called in python2. flag_displays.append('%s="%s"' % (varname, - ' '.join(_unicode_decode("%s") % (f,) for f in flags))) + ' '.join("%s" % (f,) for f in flags))) return ' '.join(flag_displays) diff --git a/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo Binary files differindex 005b007..570ef73 100644 --- a/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo +++ b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/__init__.pyo b/portage_with_autodep/pym/_emerge/__init__.pyo Binary files differindex fba4ca5..007c6ae 100644 --- a/portage_with_autodep/pym/_emerge/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo Binary files differindex 8ad61b2..0b21a14 100644 --- a/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo +++ b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo diff --git a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo Binary files differindex f211d41..16bdc11 100644 --- a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo +++ b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo diff --git a/portage_with_autodep/pym/_emerge/actions.py b/portage_with_autodep/pym/_emerge/actions.py index eaf5a15..9bb4774 100644 --- a/portage_with_autodep/pym/_emerge/actions.py +++ b/portage_with_autodep/pym/_emerge/actions.py @@ -1,7 +1,7 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals import errno import logging @@ -18,27 +18,35 @@ import sys import tempfile import textwrap import time +import warnings from itertools import chain import portage portage.proxy.lazyimport.lazyimport(globals(), + 'portage.dbapi._similar_name_search:similar_name_search', + 'portage.debug', 'portage.news:count_unread_news,display_news_notifications', + 'portage.util._get_vm_info:get_vm_info', + '_emerge.chk_updated_cfg_files:chk_updated_cfg_files', + '_emerge.help:help@emerge_help', + '_emerge.post_emerge:display_news_notification,post_emerge', + '_emerge.stdout_spinner:stdout_spinner', ) from portage.localization import _ from portage import os from portage import shutil -from portage import eapi_is_supported, _unicode_decode +from portage import eapi_is_supported, _encodings, _unicode_decode from portage.cache.cache_errors import CacheError -from portage.const import GLOBAL_CONFIG_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG +from portage.const import GLOBAL_CONFIG_PATH, VCS_DIRS, _DEPCLEAN_LIB_CHECK_DEFAULT +from portage.const import SUPPORTED_BINPKG_FORMATS, TIMESTAMP_FORMAT from portage.dbapi.dep_expand import dep_expand from portage.dbapi._expand_new_virt import expand_new_virt -from portage.dep import Atom, extended_cp_match +from portage.dep import Atom from portage.eclass_cache import hashed_path -from portage.exception import InvalidAtom +from portage.exception import InvalidAtom, InvalidData from portage.output import blue, bold, colorize, create_color_func, darkgreen, \ - red, yellow + red, xtermTitle, xtermTitleReset, yellow good = create_color_func("GOOD") bad = create_color_func("BAD") warn = create_color_func("WARN") @@ -46,9 +54,13 @@ from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.package.ebuild.doebuild import _check_temp_dir from portage._sets import load_default_config, SETPREFIX from portage._sets.base import InternalPackageSet -from portage.util import cmp_sort_key, writemsg, \ +from portage.util import cmp_sort_key, writemsg, varexpand, \ writemsg_level, writemsg_stdout from portage.util.digraph import digraph +from portage.util.SlotObject import SlotObject +from portage.util._async.run_main_scheduler import run_main_scheduler +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.global_event_loop import global_event_loop from portage._global_updates import _global_updates from _emerge.clear_caches import clear_caches @@ -76,6 +88,9 @@ from _emerge.userquery import userquery if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode def action_build(settings, trees, mtimedb, myopts, myaction, myfiles, spinner): @@ -172,6 +187,7 @@ def action_build(settings, trees, mtimedb, verbose = "--verbose" in myopts quiet = "--quiet" in myopts myparams = create_depgraph_params(myopts, myaction) + mergelist_shown = False if pretend or fetchonly: # make the mtimedb readonly @@ -273,8 +289,14 @@ def action_build(settings, trees, mtimedb, "dropped due to\n" + \ "!!! masking or unsatisfied dependencies:\n\n", noiselevel=-1) - for task in dropped_tasks: - portage.writemsg(" " + str(task) + "\n", noiselevel=-1) + for task, atoms in dropped_tasks.items(): + if not atoms: + writemsg(" %s is masked or unavailable\n" % + (task,), noiselevel=-1) + else: + writemsg(" %s requires %s\n" % + (task, ", ".join(atoms)), noiselevel=-1) + portage.writemsg("\n", noiselevel=-1) del dropped_tasks else: @@ -305,6 +327,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.display_problems() return 1 + mergecount = None if "--pretend" not in myopts and \ ("--ask" in myopts or "--tree" in myopts or \ "--verbose" in myopts) and \ @@ -316,17 +339,19 @@ def action_build(settings, trees, mtimedb, return os.EX_OK favorites = mtimedb["resume"]["favorites"] retval = mydepgraph.display( - mydepgraph.altlist(reversed=tree), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval prompt="Would you like to resume merging these packages?" else: retval = mydepgraph.display( - mydepgraph.altlist(reversed=("--tree" in myopts)), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval mergecount=0 @@ -334,6 +359,7 @@ def action_build(settings, trees, mtimedb, if isinstance(x, Package) and x.operation == "merge": mergecount += 1 + prompt = None if mergecount==0: sets = trees[settings['EROOT']]['root_config'].sets world_candidates = None @@ -346,14 +372,11 @@ def action_build(settings, trees, mtimedb, world_candidates = [x for x in favorites \ if not (x.startswith(SETPREFIX) and \ not sets[x[1:]].world_candidate)] + if "selective" in myparams and \ not oneshot and world_candidates: - print() - for x in world_candidates: - print(" %s %s" % (good("*"), x)) - prompt="Would you like to add these packages to your world favorites?" - elif settings["AUTOCLEAN"] and "yes"==settings["AUTOCLEAN"]: - prompt="Nothing to merge; would you like to auto-clean packages?" + # Prompt later, inside saveNomergeFavorites. + prompt = None else: print() print("Nothing to merge; quitting.") @@ -364,13 +387,15 @@ def action_build(settings, trees, mtimedb, else: prompt="Would you like to merge these packages?" print() - if "--ask" in myopts and userquery(prompt, enter_invalid) == "No": + if prompt is not None and "--ask" in myopts and \ + userquery(prompt, enter_invalid) == "No": print() print("Quitting.") print() return 128 + signal.SIGINT # Don't ask again (e.g. when auto-cleaning packages after merge) - myopts.pop("--ask", None) + if mergecount != 0: + myopts.pop("--ask", None) if ("--pretend" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts): if ("--resume" in myopts): @@ -380,45 +405,27 @@ def action_build(settings, trees, mtimedb, return os.EX_OK favorites = mtimedb["resume"]["favorites"] retval = mydepgraph.display( - mydepgraph.altlist(reversed=tree), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval else: retval = mydepgraph.display( - mydepgraph.altlist(reversed=("--tree" in myopts)), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval - if "--buildpkgonly" in myopts: - graph_copy = mydepgraph._dynamic_config.digraph.copy() - removed_nodes = set() - for node in graph_copy: - if not isinstance(node, Package) or \ - node.operation == "nomerge": - removed_nodes.add(node) - graph_copy.difference_update(removed_nodes) - if not graph_copy.hasallzeros(ignore_priority = \ - DepPrioritySatisfiedRange.ignore_medium): - print("\n!!! --buildpkgonly requires all dependencies to be merged.") - print("!!! You have to merge the dependencies before you can build this package.\n") - return 1 + else: - if "--buildpkgonly" in myopts: - graph_copy = mydepgraph._dynamic_config.digraph.copy() - removed_nodes = set() - for node in graph_copy: - if not isinstance(node, Package) or \ - node.operation == "nomerge": - removed_nodes.add(node) - graph_copy.difference_update(removed_nodes) - if not graph_copy.hasallzeros(ignore_priority = \ - DepPrioritySatisfiedRange.ignore_medium): - print("\n!!! --buildpkgonly requires all dependencies to be merged.") - print("!!! Cannot merge requested packages. Merge deps and try again.\n") - return 1 + + if not mergelist_shown: + # If we haven't already shown the merge list above, at + # least show warnings about missed updates and such. + mydepgraph.display_problems() if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] @@ -433,25 +440,29 @@ def action_build(settings, trees, mtimedb, mydepgraph.saveNomergeFavorites() - mergetask = Scheduler(settings, trees, mtimedb, myopts, - spinner, favorites=favorites, - graph_config=mydepgraph.schedulerGraph()) - - del mydepgraph - clear_caches(trees) - - retval = mergetask.merge() - - if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend): - if "yes" == settings.get("AUTOCLEAN"): - portage.writemsg_stdout(">>> Auto-cleaning packages...\n") - unmerge(trees[settings['EROOT']]['root_config'], - myopts, "clean", [], - ldpath_mtimes, autoclean=1) - else: - portage.writemsg_stdout(colorize("WARN", "WARNING:") - + " AUTOCLEAN is disabled. This can cause serious" - + " problems due to overlapping packages.\n") + if mergecount == 0: + retval = os.EX_OK + else: + mergetask = Scheduler(settings, trees, mtimedb, myopts, + spinner, favorites=favorites, + graph_config=mydepgraph.schedulerGraph()) + + del mydepgraph + clear_caches(trees) + + retval = mergetask.merge() + + if retval == os.EX_OK and \ + not (buildpkgonly or fetchonly or pretend): + if "yes" == settings.get("AUTOCLEAN"): + portage.writemsg_stdout(">>> Auto-cleaning packages...\n") + unmerge(trees[settings['EROOT']]['root_config'], + myopts, "clean", [], + ldpath_mtimes, autoclean=1) + else: + portage.writemsg_stdout(colorize("WARN", "WARNING:") + + " AUTOCLEAN is disabled. This can cause serious" + + " problems due to overlapping packages.\n") return retval @@ -531,7 +542,8 @@ def action_depclean(settings, trees, ldpath_mtimes, # specific packages. msg = [] - if not _ENABLE_DYN_LINK_MAP: + if "preserve-libs" not in settings.features and \ + not myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n": msg.append("Depclean may break link level dependencies. Thus, it is\n") msg.append("recommended to use a tool such as " + good("`revdep-rebuild`") + " (from\n") msg.append("app-portage/gentoolkit) in order to detect such breakage.\n") @@ -597,11 +609,17 @@ def action_depclean(settings, trees, ldpath_mtimes, if not cleanlist and "--quiet" in myopts: return rval + set_atoms = {} + for k in ("system", "selected"): + try: + set_atoms[k] = root_config.setconfig.getSetAtoms(k) + except portage.exception.PackageSetNotFound: + # A nested set could not be resolved, so ignore nested sets. + set_atoms[k] = root_config.sets[k].getAtoms() + print("Packages installed: " + str(len(vardb.cpv_all()))) - print("Packages in world: " + \ - str(len(root_config.sets["selected"].getAtoms()))) - print("Packages in system: " + \ - str(len(root_config.sets["system"].getAtoms()))) + print("Packages in world: %d" % len(set_atoms["selected"])) + print("Packages in system: %d" % len(set_atoms["system"])) print("Required packages: "+str(req_pkg_count)) if "--pretend" in myopts: print("Number to remove: "+str(len(cleanlist))) @@ -634,13 +652,21 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_sets[protected_set_name] = protected_set system_set = psets["system"] - if not system_set or not selected_set: + set_atoms = {} + for k in ("system", "selected"): + try: + set_atoms[k] = root_config.setconfig.getSetAtoms(k) + except portage.exception.PackageSetNotFound: + # A nested set could not be resolved, so ignore nested sets. + set_atoms[k] = root_config.sets[k].getAtoms() - if not system_set: + if not set_atoms["system"] or not set_atoms["selected"]: + + if not set_atoms["system"]: writemsg_level("!!! You have no system list.\n", level=logging.ERROR, noiselevel=-1) - if not selected_set: + if not set_atoms["selected"]: writemsg_level("!!! You have no world file.\n", level=logging.WARNING, noiselevel=-1) @@ -684,7 +710,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -738,7 +764,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -756,7 +782,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_sets['__excluded__'].add("=" + pkg.cpv) except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e required_sets['__excluded__'].add("=" + pkg.cpv) @@ -792,7 +818,12 @@ def calc_depclean(settings, trees, ldpath_mtimes, msg.append("the following required packages not being installed:") msg.append("") for atom, parent in unresolvable: - msg.append(" %s pulled in by:" % (atom,)) + if atom != atom.unevaluated_atom and \ + vardb.match(_unicode(atom)): + msg.append(" %s (%s) pulled in by:" % + (atom.unevaluated_atom, atom)) + else: + msg.append(" %s pulled in by:" % (atom,)) msg.append(" %s" % (parent,)) msg.append("") msg.extend(textwrap.wrap( @@ -835,15 +866,27 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_pkgs_total += 1 def show_parents(child_node): - parent_nodes = graph.parent_nodes(child_node) - if not parent_nodes: + parent_atoms = \ + resolver._dynamic_config._parent_atoms.get(child_node, []) + + # Never display the special internal protected_set. + parent_atoms = [parent_atom for parent_atom in parent_atoms + if not (isinstance(parent_atom[0], SetArg) and + parent_atom[0].name == protected_set_name)] + + if not parent_atoms: # With --prune, the highest version can be pulled in without any # real parent since all installed packages are pulled in. In that # case there's nothing to show here. return + parent_atom_dict = {} + for parent, atom in parent_atoms: + parent_atom_dict.setdefault(parent, []).append(atom) + parent_strs = [] - for node in parent_nodes: - parent_strs.append(str(getattr(node, "cpv", node))) + for parent, atoms in parent_atom_dict.items(): + parent_strs.append("%s requires %s" % + (getattr(parent, "cpv", parent), ", ".join(atoms))) parent_strs.sort() msg = [] msg.append(" %s pulled in by:\n" % (child_node.cpv,)) @@ -868,12 +911,6 @@ def calc_depclean(settings, trees, ldpath_mtimes, graph.debug_print() writemsg("\n", noiselevel=-1) - # Never display the special internal protected_set. - for node in graph: - if isinstance(node, SetArg) and node.name == protected_set_name: - graph.remove(node) - break - pkgs_to_remove = [] if action == "depclean": @@ -926,10 +963,19 @@ def calc_depclean(settings, trees, ldpath_mtimes, cleanlist = create_cleanlist() clean_set = set(cleanlist) - if cleanlist and \ - real_vardb._linkmap is not None and \ - myopts.get("--depclean-lib-check") != "n" and \ - "preserve-libs" not in settings.features: + depclean_lib_check = cleanlist and real_vardb._linkmap is not None and \ + myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n" + preserve_libs = "preserve-libs" in settings.features + preserve_libs_restrict = False + + if depclean_lib_check and preserve_libs: + for pkg in cleanlist: + if "preserve-libs" in pkg.restrict: + preserve_libs_restrict = True + break + + if depclean_lib_check and \ + (preserve_libs_restrict or not preserve_libs): # Check if any of these packages are the sole providers of libraries # with consumers that have not been selected for removal. If so, these @@ -942,6 +988,13 @@ def calc_depclean(settings, trees, ldpath_mtimes, writemsg_level(">>> Checking for lib consumers...\n") for pkg in cleanlist: + + if preserve_libs and "preserve-libs" not in pkg.restrict: + # Any needed libraries will be preserved + # when this package is unmerged, so there's + # no need to account for it here. + continue + pkg_dblink = real_vardb._dblink(pkg.cpv) consumers = {} @@ -1096,7 +1149,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, "installed", root_config, installed=True) if not resolver._add_pkg(pkg, Dependency(parent=consumer_pkg, - priority=UnmergeDepPriority(runtime=True), + priority=UnmergeDepPriority(runtime=True, + runtime_slot_op=True), root=pkg.root)): resolver.display_problems() return 1, [], False, 0 @@ -1133,30 +1187,30 @@ def calc_depclean(settings, trees, ldpath_mtimes, graph = digraph() del cleanlist[:] - dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] runtime = UnmergeDepPriority(runtime=True) runtime_post = UnmergeDepPriority(runtime_post=True) buildtime = UnmergeDepPriority(buildtime=True) priority_map = { "RDEPEND": runtime, "PDEPEND": runtime_post, + "HDEPEND": buildtime, "DEPEND": buildtime, } for node in clean_set: graph.add(node, None) - for dep_type in dep_keys: - depstr = node.metadata[dep_type] + for dep_type in Package._dep_keys: + depstr = node._metadata[dep_type] if not depstr: continue priority = priority_map[dep_type] if debug: - writemsg_level(_unicode_decode("\nParent: %s\n") \ + writemsg_level("\nParent: %s\n" % (node,), noiselevel=-1, level=logging.DEBUG) - writemsg_level(_unicode_decode( "Depstring: %s\n") \ + writemsg_level( "Depstring: %s\n" % (depstr,), noiselevel=-1, level=logging.DEBUG) - writemsg_level(_unicode_decode( "Priority: %s\n") \ + writemsg_level( "Priority: %s\n" % (priority,), noiselevel=-1, level=logging.DEBUG) try: @@ -1170,7 +1224,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, if debug: writemsg_level("Candidates: [%s]\n" % \ - ', '.join(_unicode_decode("'%s'") % (x,) for x in atoms), + ', '.join("'%s'" % (x,) for x in atoms), noiselevel=-1, level=logging.DEBUG) for atom in atoms: @@ -1184,7 +1238,15 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue for child_node in matches: if child_node in clean_set: - graph.add(child_node, node, priority=priority) + + mypriority = priority.copy() + if atom.slot_operator_built: + if mypriority.buildtime: + mypriority.buildtime_slot_op = True + if mypriority.runtime: + mypriority.runtime_slot_op = True + + graph.add(child_node, node, priority=mypriority) if debug: writemsg_level("\nunmerge digraph:\n\n", @@ -1264,11 +1326,8 @@ def action_deselect(settings, trees, opts, atoms): allow_repo=True, allow_wildcard=True)) for cpv in vardb.match(atom): - slot, = vardb.aux_get(cpv, ["SLOT"]) - if not slot: - slot = "0" - expanded_atoms.add(Atom("%s:%s" % \ - (portage.cpv_getkey(cpv), slot))) + pkg = vardb._pkg_str(cpv, None) + expanded_atoms.add(Atom("%s:%s" % (pkg.cp, pkg.slot))) discard_atoms = set() for atom in world_set: @@ -1287,12 +1346,21 @@ def action_deselect(settings, trees, opts, atoms): break if discard_atoms: for atom in sorted(discard_atoms): + if pretend: - print(">>> Would remove %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + action_desc = "Would remove" + else: + action_desc = "Removing" + + if atom.startswith(SETPREFIX): + filename = "world_sets" else: - print(">>> Removing %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + filename = "world" + + writemsg_stdout( + ">>> %s %s from \"%s\" favorites file...\n" % + (action_desc, colorize("INFORM", _unicode(atom)), + filename), noiselevel=-1) if '--ask' in opts: prompt = "Would you like to remove these " + \ @@ -1330,10 +1398,90 @@ class _info_pkgs_ver(object): def action_info(settings, trees, myopts, myfiles): + # See if we can find any packages installed matching the strings + # passed on the command line + mypkgs = [] + eroot = settings['EROOT'] + vardb = trees[eroot]["vartree"].dbapi + portdb = trees[eroot]['porttree'].dbapi + bindb = trees[eroot]["bintree"].dbapi + for x in myfiles: + any_match = False + cp_exists = bool(vardb.match(x.cp)) + installed_match = vardb.match(x) + for installed in installed_match: + mypkgs.append((installed, "installed")) + any_match = True + + if any_match: + continue + + for db, pkg_type in ((portdb, "ebuild"), (bindb, "binary")): + if pkg_type == "binary" and "--usepkg" not in myopts: + continue + + # Use match instead of cp_list, to account for old-style virtuals. + if not cp_exists and db.match(x.cp): + cp_exists = True + # Search for masked packages too. + if not cp_exists and hasattr(db, "xmatch") and \ + db.xmatch("match-all", x.cp): + cp_exists = True + + matches = db.match(x) + matches.reverse() + for match in matches: + if pkg_type == "binary": + if db.bintree.isremote(match): + continue + auxkeys = ["EAPI", "DEFINED_PHASES"] + metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys))) + if metadata["EAPI"] not in ("0", "1", "2", "3") and \ + "info" in metadata["DEFINED_PHASES"].split(): + mypkgs.append((match, pkg_type)) + break + + if not cp_exists: + xinfo = '"%s"' % x.unevaluated_atom + # Discard null/ from failed cpv_expand category expansion. + xinfo = xinfo.replace("null/", "") + if settings["ROOT"] != "/": + xinfo = "%s for %s" % (xinfo, eroot) + writemsg("\nemerge: there are no ebuilds to satisfy %s.\n" % + colorize("INFORM", xinfo), noiselevel=-1) + + if myopts.get("--misspell-suggestions", "y") != "n": + + writemsg("\nemerge: searching for similar names..." + , noiselevel=-1) + + dbs = [vardb] + #if "--usepkgonly" not in myopts: + dbs.append(portdb) + if "--usepkg" in myopts: + dbs.append(bindb) + + matches = similar_name_search(dbs, x) + + if len(matches) == 1: + writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" + , noiselevel=-1) + elif len(matches) > 1: + writemsg( + "\nemerge: Maybe you meant any of these: %s?\n" % \ + (", ".join(matches),), noiselevel=-1) + else: + # Generally, this would only happen if + # all dbapis are empty. + writemsg(" nothing similar found.\n" + , noiselevel=-1) + + return 1 + output_buffer = [] append = output_buffer.append root_config = trees[settings['EROOT']]['root_config'] - running_eroot = trees._running_eroot + chost = settings.get("CHOST") append(getportageversion(settings["PORTDIR"], None, settings.profile_path, settings["CHOST"], @@ -1347,6 +1495,18 @@ def action_info(settings, trees, myopts, myfiles): append(header_width * "=") append("System uname: %s" % (platform.platform(aliased=1),)) + vm_info = get_vm_info() + if "ram.total" in vm_info: + line = "%-9s %10d total" % ("KiB Mem:", vm_info["ram.total"] / 1024) + if "ram.free" in vm_info: + line += ",%10d free" % (vm_info["ram.free"] / 1024,) + append(line) + if "swap.total" in vm_info: + line = "%-9s %10d total" % ("KiB Swap:", vm_info["swap.total"] / 1024) + if "swap.free" in vm_info: + line += ",%10d free" % (vm_info["swap.free"] / 1024,) + append(line) + lastSync = portage.grabfile(os.path.join( settings["PORTDIR"], "metadata", "timestamp.chk")) if lastSync: @@ -1355,6 +1515,23 @@ def action_info(settings, trees, myopts, myfiles): lastSync = "Unknown" append("Timestamp of tree: %s" % (lastSync,)) + ld_names = [] + if chost: + ld_names.append(chost + "-ld") + ld_names.append("ld") + for name in ld_names: + try: + proc = subprocess.Popen([name, "--version"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + pass + else: + output = _unicode_decode(proc.communicate()[0]).splitlines() + proc.wait() + if proc.wait() == os.EX_OK and output: + append("ld %s" % (output[0])) + break + try: proc = subprocess.Popen(["distcc", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -1391,7 +1568,6 @@ def action_info(settings, trees, myopts, myfiles): "sys-devel/binutils", "sys-devel/libtool", "dev-lang/python"] myvars += portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_pkgs") atoms = [] - vardb = trees[running_eroot]['vartree'].dbapi for x in myvars: try: x = Atom(x) @@ -1404,7 +1580,6 @@ def action_info(settings, trees, myopts, myfiles): myvars = sorted(set(atoms)) - portdb = trees[running_eroot]['porttree'].dbapi main_repo = portdb.getRepositoryName(portdb.porttree_root) cp_map = {} cp_max_len = 0 @@ -1456,11 +1631,11 @@ def action_info(settings, trees, myopts, myfiles): append("Repositories: %s" % \ " ".join(repo.name for repo in repos)) - if _ENABLE_SET_CONFIG: + installed_sets = sorted(s for s in + root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX)) + if installed_sets: sets_line = "Installed sets: " - sets_line += ", ".join(s for s in \ - sorted(root_config.sets['selected'].getNonAtoms()) \ - if s.startswith(SETPREFIX)) + sets_line += ", ".join(installed_sets) append(sets_line) if "--verbose" in myopts: @@ -1471,7 +1646,7 @@ def action_info(settings, trees, myopts, myfiles): 'PORTDIR_OVERLAY', 'PORTAGE_BUNZIP2_COMMAND', 'PORTAGE_BZIP2_COMMAND', 'USE', 'CHOST', 'CFLAGS', 'CXXFLAGS', - 'ACCEPT_KEYWORDS', 'ACCEPT_LICENSE', 'SYNC', 'FEATURES', + 'ACCEPT_KEYWORDS', 'ACCEPT_LICENSE', 'FEATURES', 'EMERGE_DEFAULT_OPTS'] myvars.extend(portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_vars")) @@ -1517,40 +1692,7 @@ def action_info(settings, trees, myopts, myfiles): append("") writemsg_stdout("\n".join(output_buffer), noiselevel=-1) - - # See if we can find any packages installed matching the strings - # passed on the command line - mypkgs = [] - eroot = settings['EROOT'] - vardb = trees[eroot]["vartree"].dbapi - portdb = trees[eroot]['porttree'].dbapi - bindb = trees[eroot]["bintree"].dbapi - for x in myfiles: - match_found = False - installed_match = vardb.match(x) - for installed in installed_match: - mypkgs.append((installed, "installed")) - match_found = True - - if match_found: - continue - - for db, pkg_type in ((portdb, "ebuild"), (bindb, "binary")): - if pkg_type == "binary" and "--usepkg" not in myopts: - continue - - matches = db.match(x) - matches.reverse() - for match in matches: - if pkg_type == "binary": - if db.bintree.isremote(match): - continue - auxkeys = ["EAPI", "DEFINED_PHASES"] - metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys))) - if metadata["EAPI"] not in ("0", "1", "2", "3") and \ - "info" in metadata["DEFINED_PHASES"].split(): - mypkgs.append((match, pkg_type)) - break + del output_buffer[:] # If some packages were found... if mypkgs: @@ -1564,11 +1706,15 @@ def action_info(settings, trees, myopts, myfiles): # Loop through each package # Only print settings if they differ from global settings header_title = "Package Settings" - print(header_width * "=") - print(header_title.rjust(int(header_width/2 + len(header_title)/2))) - print(header_width * "=") - from portage.output import EOutput - out = EOutput() + append(header_width * "=") + append(header_title.rjust(int(header_width/2 + len(header_title)/2))) + append(header_width * "=") + append("") + writemsg_stdout("\n".join(output_buffer), + noiselevel=-1) + del output_buffer[:] + + out = portage.output.EOutput() for mypkg in mypkgs: cpv = mypkg[0] pkg_type = mypkg[1] @@ -1586,28 +1732,32 @@ def action_info(settings, trees, myopts, myfiles): root_config=root_config, type_name=pkg_type) if pkg_type == "installed": - print("\n%s was built with the following:" % \ + append("\n%s was built with the following:" % \ colorize("INFORM", str(pkg.cpv))) elif pkg_type == "ebuild": - print("\n%s would be build with the following:" % \ + append("\n%s would be build with the following:" % \ colorize("INFORM", str(pkg.cpv))) elif pkg_type == "binary": - print("\n%s (non-installed binary) was built with the following:" % \ + append("\n%s (non-installed binary) was built with the following:" % \ colorize("INFORM", str(pkg.cpv))) - writemsg_stdout('%s\n' % pkg_use_display(pkg, myopts), - noiselevel=-1) + append('%s' % pkg_use_display(pkg, myopts)) if pkg_type == "installed": for myvar in mydesiredvars: if metadata[myvar].split() != settings.get(myvar, '').split(): - print("%s=\"%s\"" % (myvar, metadata[myvar])) - print() + append("%s=\"%s\"" % (myvar, metadata[myvar])) + append("") + append("") + writemsg_stdout("\n".join(output_buffer), + noiselevel=-1) + del output_buffer[:] if metadata['DEFINED_PHASES']: if 'info' not in metadata['DEFINED_PHASES'].split(): continue - print(">>> Attempting to run pkg_info() for '%s'" % pkg.cpv) + writemsg_stdout(">>> Attempting to run pkg_info() for '%s'\n" + % pkg.cpv, noiselevel=-1) if pkg_type == "installed": ebuildpath = vardb.findname(pkg.cpv) @@ -1834,6 +1984,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): print() signal.signal(signal.SIGWINCH, signal.SIG_DFL) + portdb.flush_cache() sys.stdout.flush() os.umask(old_umask) @@ -1843,35 +1994,12 @@ def action_regen(settings, portdb, max_jobs, max_load): #regenerate cache entries sys.stdout.flush() - regen = MetadataRegen(portdb, max_jobs=max_jobs, max_load=max_load) - received_signal = [] + regen = MetadataRegen(portdb, max_jobs=max_jobs, + max_load=max_load, main=True) - def emergeexitsig(signum, frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % \ - {"signal":signum}) - regen.terminate() - received_signal.append(128 + signum) - - earlier_sigint_handler = signal.signal(signal.SIGINT, emergeexitsig) - earlier_sigterm_handler = signal.signal(signal.SIGTERM, emergeexitsig) - - try: - regen.run() - finally: - # Restore previous handlers - if earlier_sigint_handler is not None: - signal.signal(signal.SIGINT, earlier_sigint_handler) - else: - signal.signal(signal.SIGINT, signal.SIG_DFL) - if earlier_sigterm_handler is not None: - signal.signal(signal.SIGTERM, earlier_sigterm_handler) - else: - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - if received_signal: - sys.exit(received_signal[0]) + signum = run_main_scheduler(regen) + if signum is not None: + sys.exit(128 + signum) portage.writemsg_stdout("done!\n") return regen.returncode @@ -1892,37 +2020,110 @@ def action_search(root_config, myopts, myfiles, spinner): sys.exit(1) searchinstance.output() -def action_sync(settings, trees, mtimedb, myopts, myaction): +def action_sync(emerge_config, trees=DeprecationWarning, + mtimedb=DeprecationWarning, opts=DeprecationWarning, + action=DeprecationWarning): + + if not isinstance(emerge_config, _emerge_config): + warnings.warn("_emerge.actions.action_sync() now expects " + "an _emerge_config instance as the first parameter", + DeprecationWarning, stacklevel=2) + emerge_config = load_emerge_config( + action=action, args=[], trees=trees, opts=opts) + + xterm_titles = "notitles" not in \ + emerge_config.target_config.settings.features + emergelog(xterm_titles, " === sync") + + selected_repos = [] + unknown_repo_names = [] + missing_sync_type = [] + if emerge_config.args: + for repo_name in emerge_config.args: + try: + repo = emerge_config.target_config.settings.repositories[repo_name] + except KeyError: + unknown_repo_names.append(repo_name) + else: + selected_repos.append(repo) + if repo.sync_type is None: + missing_sync_type.append(repo) + + if unknown_repo_names: + writemsg_level("!!! %s\n" % _("Unknown repo(s): %s") % + " ".join(unknown_repo_names), + level=logging.ERROR, noiselevel=-1) + + if missing_sync_type: + writemsg_level("!!! %s\n" % + _("Missing sync-type for repo(s): %s") % + " ".join(repo.name for repo in missing_sync_type), + level=logging.ERROR, noiselevel=-1) + + if unknown_repo_names or missing_sync_type: + return 1 + + else: + selected_repos.extend(emerge_config.target_config.settings.repositories) + + for repo in selected_repos: + if repo.sync_type is not None: + returncode = _sync_repo(emerge_config, repo) + if returncode != os.EX_OK: + return returncode + + # Reload the whole config from scratch. + portage._sync_disabled_warnings = False + load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) + + if emerge_config.opts.get('--package-moves') != 'n' and \ + _global_updates(emerge_config.trees, + emerge_config.target_config.mtimedb["updates"], + quiet=("--quiet" in emerge_config.opts)): + emerge_config.target_config.mtimedb.commit() + # Reload the whole config from scratch. + load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) + + mybestpv = emerge_config.target_config.trees['porttree'].dbapi.xmatch( + "bestmatch-visible", portage.const.PORTAGE_PACKAGE_ATOM) + mypvs = portage.best( + emerge_config.target_config.trees['vartree'].dbapi.match( + portage.const.PORTAGE_PACKAGE_ATOM)) + + chk_updated_cfg_files(emerge_config.target_config.root, + portage.util.shlex_split( + emerge_config.target_config.settings.get("CONFIG_PROTECT", ""))) + + if mybestpv != mypvs and "--quiet" not in emerge_config.opts: + print() + print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") + print(warn(" * ")+"that you update portage now, before any other packages are updated.") + print() + print(warn(" * ")+"To update portage, run 'emerge --oneshot portage' now.") + print() + + display_news_notification(emerge_config.target_config, emerge_config.opts) + return os.EX_OK + +def _sync_repo(emerge_config, repo): + settings, trees, mtimedb = emerge_config + myopts = emerge_config.opts enter_invalid = '--ask-enter-invalid' in myopts xterm_titles = "notitles" not in settings.features - emergelog(xterm_titles, " === sync") - portdb = trees[settings['EROOT']]['porttree'].dbapi - myportdir = portdb.porttree_root - if not myportdir: - myportdir = settings.get('PORTDIR', '') - if myportdir and myportdir.strip(): - myportdir = os.path.realpath(myportdir) - else: - myportdir = None + msg = ">>> Synchronization of repository '%s' located in '%s'..." % (repo.name, repo.location) + emergelog(xterm_titles, msg) + writemsg_level(msg + "\n") out = portage.output.EOutput() - global_config_path = GLOBAL_CONFIG_PATH - if settings['EPREFIX']: - global_config_path = os.path.join(settings['EPREFIX'], - GLOBAL_CONFIG_PATH.lstrip(os.sep)) - if not myportdir: - sys.stderr.write("!!! PORTDIR is undefined. " + \ - "Is %s/make.globals missing?\n" % global_config_path) - sys.exit(1) - if myportdir[-1]=="/": - myportdir=myportdir[:-1] try: - st = os.stat(myportdir) + st = os.stat(repo.location) except OSError: st = None if st is None: - print(">>>",myportdir,"not found, creating it.") - portage.util.ensure_dirs(myportdir, mode=0o755) - st = os.stat(myportdir) + print(">>> '%s' not found, creating it." % repo.location) + portage.util.ensure_dirs(repo.location, mode=0o755) + st = os.stat(repo.location) usersync_uid = None spawn_kwargs = {} @@ -1955,59 +2156,51 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if rval != os.EX_OK: return rval - syncuri = settings.get("SYNC", "").strip() - if not syncuri: - writemsg_level("!!! SYNC is undefined. " + \ - "Is %s/make.globals missing?\n" % global_config_path, - noiselevel=-1, level=logging.ERROR) - return 1 + syncuri = repo.sync_uri - vcs_dirs = frozenset([".git", ".svn", "CVS", ".hg"]) - vcs_dirs = vcs_dirs.intersection(os.listdir(myportdir)) + vcs_dirs = frozenset(VCS_DIRS) + vcs_dirs = vcs_dirs.intersection(os.listdir(repo.location)) os.umask(0o022) dosyncuri = syncuri updatecache_flg = False - git = False - if myaction == "metadata": - print("skipping sync") - updatecache_flg = True - elif ".git" in vcs_dirs: + if repo.sync_type == "git": # Update existing git repository, and ignore the syncuri. We are # going to trust the user and assume that the user is in the branch # that he/she wants updated. We'll let the user manage branches with # git directly. if portage.process.find_binary("git") is None: msg = ["Command not found: git", - "Type \"emerge dev-util/git\" to enable git support."] + "Type \"emerge %s\" to enable git support." % portage.const.GIT_PACKAGE_ATOM] for l in msg: writemsg_level("!!! %s\n" % l, level=logging.ERROR, noiselevel=-1) return 1 - msg = ">>> Starting git pull in %s..." % myportdir + msg = ">>> Starting git pull in %s..." % repo.location emergelog(xterm_titles, msg ) writemsg_level(msg + "\n") exitcode = portage.process.spawn_bash("cd %s ; git pull" % \ - (portage._shell_quote(myportdir),), **spawn_kwargs) + (portage._shell_quote(repo.location),), + **portage._native_kwargs(spawn_kwargs)) if exitcode != os.EX_OK: - msg = "!!! git pull error in %s." % myportdir + msg = "!!! git pull error in %s." % repo.location emergelog(xterm_titles, msg) writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) return exitcode - msg = ">>> Git pull in %s successful" % myportdir + msg = ">>> Git pull in %s successful" % repo.location emergelog(xterm_titles, msg) writemsg_level(msg + "\n") - git = True - elif syncuri[:8]=="rsync://" or syncuri[:6]=="ssh://": + elif repo.sync_type == "rsync": for vcs_dir in vcs_dirs: writemsg_level(("!!! %s appears to be under revision " + \ "control (contains %s).\n!!! Aborting rsync sync.\n") % \ - (myportdir, vcs_dir), level=logging.ERROR, noiselevel=-1) + (repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1) return 1 - if not os.path.exists("/usr/bin/rsync"): + rsync_binary = portage.process.find_binary("rsync") + if rsync_binary is None: print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.") - print("!!! Type \"emerge net-misc/rsync\" to enable rsync support.") - sys.exit(1) + print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM) + return os.EX_UNAVAILABLE mytimeout=180 rsync_opts = [] @@ -2019,6 +2212,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): "--safe-links", # Ignore links outside of tree "--perms", # Preserve permissions "--times", # Preserive mod times + "--omit-dir-times", "--compress", # Compress the data transmitted "--force", # Force deletion on non-empty dirs "--whole-file", # Don't do block transfers, only entire files @@ -2081,14 +2275,14 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # Real local timestamp file. servertimestampfile = os.path.join( - myportdir, "metadata", "timestamp.chk") + repo.location, "metadata", "timestamp.chk") content = portage.util.grabfile(servertimestampfile) mytimestamp = 0 if content: try: mytimestamp = time.mktime(time.strptime(content[0], - "%a, %d %b %Y %H:%M:%S +0000")) + TIMESTAMP_FORMAT)) except (OverflowError, ValueError): pass del content @@ -2112,9 +2306,12 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", syncuri, maxsplit=4)[1:5] except ValueError: - writemsg_level("!!! SYNC is invalid: %s\n" % syncuri, + writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri, noiselevel=-1, level=logging.ERROR) return 1 + + ssh_opts = settings.get("PORTAGE_SSH_OPTS") + if port is None: port="" if user_name is None: @@ -2230,7 +2427,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if mytimestamp != 0 and "--quiet" not in myopts: print(">>> Checking server timestamp ...") - rsynccommand = ["/usr/bin/rsync"] + rsync_opts + extra_rsync_opts + rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts + + if proto == 'ssh' and ssh_opts: + rsynccommand.append("--rsh=ssh " + ssh_opts) if "--debug" in myopts: print(rsynccommand) @@ -2276,7 +2476,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): rsync_initial_timeout) mypids.extend(portage.process.spawn( - mycommand, returnpid=True, **spawn_kwargs)) + mycommand, returnpid=True, + **portage._native_kwargs(spawn_kwargs))) exitcode = os.waitpid(mypids[0], 0)[1] if usersync_uid is not None: portage.util.apply_permissions(tmpservertimestampfile, @@ -2306,12 +2507,11 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): exitcode = (exitcode & 0xff) << 8 else: exitcode = exitcode >> 8 - if mypids: - portage.process.spawned_pids.remove(mypids[0]) + if content: try: servertimestamp = time.mktime(time.strptime( - content[0], "%a, %d %b %Y %H:%M:%S +0000")) + content[0], TIMESTAMP_FORMAT)) except (OverflowError, ValueError): pass del mycommand, mypids, content @@ -2327,7 +2527,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): print(">>> In order to force sync, remove '%s'." % servertimestampfile) print(">>>") print() - sys.exit(0) + return os.EX_OK elif (servertimestamp != 0) and (servertimestamp < mytimestamp): emergelog(xterm_titles, ">>> Server out of date: %s" % dosyncuri) @@ -2341,8 +2541,33 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): exitcode = SERVER_OUT_OF_DATE elif (servertimestamp == 0) or (servertimestamp > mytimestamp): # actual sync - mycommand = rsynccommand + [dosyncuri+"/", myportdir] - exitcode = portage.process.spawn(mycommand, **spawn_kwargs) + mycommand = rsynccommand + [dosyncuri+"/", repo.location] + exitcode = None + try: + exitcode = portage.process.spawn(mycommand, + **portage._native_kwargs(spawn_kwargs)) + finally: + if exitcode is None: + # interrupted + exitcode = 128 + signal.SIGINT + + # 0 Success + # 1 Syntax or usage error + # 2 Protocol incompatibility + # 5 Error starting client-server protocol + # 35 Timeout waiting for daemon connection + if exitcode not in (0, 1, 2, 5, 35): + # If the exit code is not among those listed above, + # then we may have a partial/inconsistent sync + # state, so our previously read timestamp as well + # as the corresponding file can no longer be + # trusted. + mytimestamp = 0 + try: + os.unlink(servertimestampfile) + except OSError: + pass + if exitcode in [0,1,3,4,11,14,20,21]: break elif exitcode in [1,3,4,11,14,20,21]: @@ -2368,23 +2593,23 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if (exitcode==0): emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri) elif exitcode == SERVER_OUT_OF_DATE: - sys.exit(1) + return 1 elif exitcode == EXCEEDED_MAX_RETRIES: sys.stderr.write( ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries) - sys.exit(1) + return 1 elif (exitcode>0): msg = [] if exitcode==1: msg.append("Rsync has reported that there is a syntax error. Please ensure") - msg.append("that your SYNC statement is proper.") - msg.append("SYNC=" + settings["SYNC"]) + msg.append("that sync-uri attribute for repository '%s' is proper." % repo.name) + msg.append("sync-uri: '%s'" % repo.sync_uri) elif exitcode==11: msg.append("Rsync has reported that there is a File IO error. Normally") msg.append("this means your disk is full, but can be caused by corruption") - msg.append("on the filesystem that contains PORTDIR. Please investigate") + msg.append("on the filesystem that contains repository '%s'. Please investigate" % repo.name) msg.append("and try again after the problem has been fixed.") - msg.append("PORTDIR=" + settings["PORTDIR"]) + msg.append("Location of repository: '%s'" % repo.location) elif exitcode==20: msg.append("Rsync was killed before it finished.") else: @@ -2395,115 +2620,76 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): msg.append("(and possibly your system's filesystem) configuration.") for line in msg: out.eerror(line) - sys.exit(exitcode) - elif syncuri[:6]=="cvs://": + return exitcode + elif repo.sync_type == "cvs": if not os.path.exists("/usr/bin/cvs"): print("!!! /usr/bin/cvs does not exist, so CVS support is disabled.") - print("!!! Type \"emerge dev-vcs/cvs\" to enable CVS support.") - sys.exit(1) - cvsroot=syncuri[6:] - cvsdir=os.path.dirname(myportdir) - if not os.path.exists(myportdir+"/CVS"): + print("!!! Type \"emerge %s\" to enable CVS support." % portage.const.CVS_PACKAGE_ATOM) + return os.EX_UNAVAILABLE + cvs_root = syncuri + if cvs_root.startswith("cvs://"): + cvs_root = cvs_root[6:] + if not os.path.exists(os.path.join(repo.location, "CVS")): #initial checkout print(">>> Starting initial cvs checkout with "+syncuri+"...") - if os.path.exists(cvsdir+"/gentoo-x86"): - print("!!! existing",cvsdir+"/gentoo-x86 directory; exiting.") - sys.exit(1) try: - os.rmdir(myportdir) + os.rmdir(repo.location) except OSError as e: if e.errno != errno.ENOENT: sys.stderr.write( - "!!! existing '%s' directory; exiting.\n" % myportdir) - sys.exit(1) + "!!! existing '%s' directory; exiting.\n" % repo.location) + return 1 del e if portage.process.spawn_bash( - "cd %s; exec cvs -z0 -d %s co -P gentoo-x86" % \ - (portage._shell_quote(cvsdir), portage._shell_quote(cvsroot)), - **spawn_kwargs) != os.EX_OK: + "cd %s; exec cvs -z0 -d %s co -P -d %s %s" % + (portage._shell_quote(os.path.dirname(repo.location)), portage._shell_quote(cvs_root), + portage._shell_quote(os.path.basename(repo.location)), portage._shell_quote(repo.sync_cvs_repo)), + **portage._native_kwargs(spawn_kwargs)) != os.EX_OK: print("!!! cvs checkout error; exiting.") - sys.exit(1) - os.rename(os.path.join(cvsdir, "gentoo-x86"), myportdir) + return 1 else: #cvs update print(">>> Starting cvs update with "+syncuri+"...") retval = portage.process.spawn_bash( "cd %s; exec cvs -z0 -q update -dP" % \ - (portage._shell_quote(myportdir),), **spawn_kwargs) + (portage._shell_quote(repo.location),), + **portage._native_kwargs(spawn_kwargs)) if retval != os.EX_OK: writemsg_level("!!! cvs update error; exiting.\n", noiselevel=-1, level=logging.ERROR) - sys.exit(retval) + return retval dosyncuri = syncuri - else: - writemsg_level("!!! Unrecognized protocol: SYNC='%s'\n" % (syncuri,), - noiselevel=-1, level=logging.ERROR) - return 1 # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - adjust_configs(myopts, trees) - root_config = trees[settings['EROOT']]['root_config'] + settings, trees, mtimedb = load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) portdb = trees[settings['EROOT']]['porttree'].dbapi - if git: + if repo.sync_type == "git": # NOTE: Do this after reloading the config, in case # it did not exist prior to sync, so that the config # and portdb properly account for its existence. - exitcode = git_sync_timestamps(portdb, myportdir) + exitcode = git_sync_timestamps(portdb, repo.location) if exitcode == os.EX_OK: updatecache_flg = True - if updatecache_flg and \ - myaction != "metadata" and \ - "metadata-transfer" not in settings.features: + if updatecache_flg and "metadata-transfer" not in settings.features: updatecache_flg = False if updatecache_flg and \ - os.path.exists(os.path.join(myportdir, 'metadata', 'cache')): + os.path.exists(os.path.join(repo.location, 'metadata', 'cache')): - # Only update cache for myportdir since that's + # Only update cache for repo.location since that's # the only one that's been synced here. - action_metadata(settings, portdb, myopts, porttrees=[myportdir]) + action_metadata(settings, portdb, myopts, porttrees=[repo.location]) - if myopts.get('--package-moves') != 'n' and \ - _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)): - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - adjust_configs(myopts, trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - root_config = trees[settings['EROOT']]['root_config'] - - mybestpv = portdb.xmatch("bestmatch-visible", - portage.const.PORTAGE_PACKAGE_ATOM) - mypvs = portage.best( - trees[settings['EROOT']]['vartree'].dbapi.match( - portage.const.PORTAGE_PACKAGE_ATOM)) - - chk_updated_cfg_files(settings["EROOT"], - portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))) - - if myaction != "metadata": - postsync = os.path.join(settings["PORTAGE_CONFIGROOT"], - portage.USER_CONFIG_PATH, "bin", "post_sync") - if os.access(postsync, os.X_OK): - retval = portage.process.spawn( - [postsync, dosyncuri], env=settings.environ()) - if retval != os.EX_OK: - writemsg_level( - " %s spawn failed of %s\n" % (bad("*"), postsync,), - level=logging.ERROR, noiselevel=-1) - - if(mybestpv != mypvs) and not "--quiet" in myopts: - print() - print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") - print(warn(" * ")+"that you update portage now, before any other packages are updated.") - print() - print(warn(" * ")+"To update portage, run 'emerge portage' now.") - print() + postsync = os.path.join(settings["PORTAGE_CONFIGROOT"], portage.USER_CONFIG_PATH, "bin", "post_sync") + if os.access(postsync, os.X_OK): + retval = portage.process.spawn([postsync, dosyncuri], env=settings.environ()) + if retval != os.EX_OK: + writemsg_level(" %s spawn failed of %s\n" % (bad("*"), postsync,), + level=logging.ERROR, noiselevel=-1) - display_news_notification(root_config, myopts) return os.EX_OK def action_uninstall(settings, trees, ldpath_mtimes, @@ -2572,16 +2758,30 @@ def action_uninstall(settings, trees, ldpath_mtimes, level=logging.ERROR, noiselevel=-1) return 1 - for cp in vardb.cp_all(): - if extended_cp_match(ext_atom.cp, cp): - atom = cp + for cpv in vardb.cpv_all(): + if portage.match_from_list(ext_atom, [cpv]): + require_metadata = False + atom = portage.cpv_getkey(cpv) + if ext_atom.operator == '=*': + atom = "=" + atom + "-" + \ + portage.versions.cpv_getversion(cpv) if ext_atom.slot: atom += ":" + ext_atom.slot + require_metadata = True if ext_atom.repo: atom += "::" + ext_atom.repo + require_metadata = True + + atom = Atom(atom, allow_repo=True) + if require_metadata: + try: + cpv = vardb._pkg_str(cpv, ext_atom.repo) + except (KeyError, InvalidData): + continue + if not portage.match_from_list(atom, [cpv]): + continue - if vardb.match(atom): - valid_atoms.append(Atom(atom, allow_repo=True)) + valid_atoms.append(atom) else: msg = [] @@ -2611,13 +2811,8 @@ def action_uninstall(settings, trees, ldpath_mtimes, if owners: for cpv in owners: - slot = vardb.aux_get(cpv, ['SLOT'])[0] - if not slot: - # portage now masks packages with missing slot, but it's - # possible that one was installed by an older version - atom = portage.cpv_getkey(cpv) - else: - atom = '%s:%s' % (portage.cpv_getkey(cpv), slot) + pkg = vardb._pkg_str(cpv, None) + atom = '%s:%s' % (pkg.cp, pkg.slot) valid_atoms.append(portage.dep.Atom(atom)) else: writemsg_level(("!!! '%s' is not claimed " + \ @@ -2641,20 +2836,20 @@ def action_uninstall(settings, trees, ldpath_mtimes, if action == 'deselect': return action_deselect(settings, trees, opts, valid_atoms) - # Create a Scheduler for calls to unmerge(), in order to cause - # redirection of ebuild phase output to logs as required for - # options such as --quiet. - sched = Scheduler(settings, trees, None, opts, - spinner, uninstall_only=True) - sched._background = sched._background_mode() - sched._status_display.quiet = True - - if sched._background: - sched.settings.unlock() - sched.settings["PORTAGE_BACKGROUND"] = "1" - sched.settings.backup_changes("PORTAGE_BACKGROUND") - sched.settings.lock() - sched.pkgsettings[eroot] = portage.config(clone=sched.settings) + # Use the same logic as the Scheduler class to trigger redirection + # of ebuild pkg_prerm/postrm phase output to logs as appropriate + # for options such as --jobs, --quiet and --quiet-build. + max_jobs = opts.get("--jobs", 1) + background = (max_jobs is True or max_jobs > 1 or + "--quiet" in opts or opts.get("--quiet-build") == "y") + sched_iface = SchedulerInterface(global_event_loop(), + is_background=lambda: background) + + if background: + settings.unlock() + settings["PORTAGE_BACKGROUND"] = "1" + settings.backup_changes("PORTAGE_BACKGROUND") + settings.lock() if action in ('clean', 'unmerge') or \ (action == 'prune' and "--nodeps" in opts): @@ -2662,10 +2857,11 @@ def action_uninstall(settings, trees, ldpath_mtimes, ordered = action == 'unmerge' rval = unmerge(trees[settings['EROOT']]['root_config'], opts, action, valid_atoms, ldpath_mtimes, ordered=ordered, - scheduler=sched._sched_iface) + scheduler=sched_iface) else: rval = action_depclean(settings, trees, ldpath_mtimes, - opts, action, valid_atoms, spinner, scheduler=sched._sched_iface) + opts, action, valid_atoms, spinner, + scheduler=sched_iface) return rval @@ -2771,6 +2967,10 @@ def adjust_config(myopts, settings): settings["NOCOLOR"] = "true" settings.backup_changes("NOCOLOR") + if "--pkg-format" in myopts: + settings["PORTAGE_BINPKG_FORMAT"] = myopts["--pkg-format"] + settings.backup_changes("PORTAGE_BINPKG_FORMAT") + def display_missing_pkg_set(root_config, set_name): msg = [] @@ -2797,6 +2997,7 @@ def relative_profile_path(portdir, abs_profile): def getportageversion(portdir, _unused, profile, chost, vardb): profilever = None + repositories = vardb.settings.repositories if profile: profilever = relative_profile_path(portdir, profile) if profilever is None: @@ -2807,6 +3008,20 @@ def getportageversion(portdir, _unused, profile, chost, vardb): os.path.join(profile, parent)) if profilever is not None: break + colon = parent.find(":") + if colon != -1: + p_repo_name = parent[:colon] + try: + p_repo_loc = \ + repositories.get_location_for_name(p_repo_name) + except KeyError: + pass + else: + profilever = relative_profile_path(p_repo_loc, + os.path.join(p_repo_loc, 'profiles', + parent[colon+1:])) + if profilever is not None: + break except portage.exception.PortageException: pass @@ -2979,61 +3194,53 @@ def git_sync_timestamps(portdb, portdir): return os.EX_OK -def load_emerge_config(trees=None): +class _emerge_config(SlotObject): + + __slots__ = ('action', 'args', 'opts', + 'running_config', 'target_config', 'trees') + + # Support unpack as tuple, for load_emerge_config backward compatibility. + def __iter__(self): + yield self.target_config.settings + yield self.trees + yield self.target_config.mtimedb + + def __getitem__(self, index): + return list(self)[index] + + def __len__(self): + return 3 + +def load_emerge_config(emerge_config=None, **kargs): + + if emerge_config is None: + emerge_config = _emerge_config(**kargs) + kwargs = {} - for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): + for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT"), + ("eprefix", "EPREFIX")): v = os.environ.get(envvar, None) if v and v.strip(): kwargs[k] = v - trees = portage.create_trees(trees=trees, **kwargs) + emerge_config.trees = portage.create_trees(trees=emerge_config.trees, + **portage._native_kwargs(kwargs)) - for root_trees in trees.values(): + for root_trees in emerge_config.trees.values(): settings = root_trees["vartree"].settings settings._init_dirs() setconfig = load_default_config(settings, root_trees) root_trees["root_config"] = RootConfig(settings, root_trees, setconfig) - settings = trees[trees._target_eroot]['vartree'].settings - mtimedbfile = os.path.join(settings['EROOT'], portage.CACHE_PATH, "mtimedb") - mtimedb = portage.MtimeDB(mtimedbfile) - QueryCommand._db = trees - return settings, trees, mtimedb - -def chk_updated_cfg_files(eroot, config_protect): - target_root = eroot - result = list( - portage.util.find_updated_config_files(target_root, config_protect)) - - for x in result: - writemsg_level("\n %s " % (colorize("WARN", "* " + _("IMPORTANT:"))), - level=logging.INFO, noiselevel=-1) - if not x[1]: # it's a protected file - writemsg_level( _("config file '%s' needs updating.\n") % x[0], - level=logging.INFO, noiselevel=-1) - else: # it's a protected dir - if len(x[1]) == 1: - head, tail = os.path.split(x[1][0]) - tail = tail[len("._cfg0000_"):] - fpath = os.path.join(head, tail) - writemsg_level(_("config file '%s' needs updating.\n") % fpath, - level=logging.INFO, noiselevel=-1) - else: - writemsg_level( _("%d config files in '%s' need updating.\n") % \ - (len(x[1]), x[0]), level=logging.INFO, noiselevel=-1) + target_eroot = emerge_config.trees._target_eroot + emerge_config.target_config = \ + emerge_config.trees[target_eroot]['root_config'] + emerge_config.target_config.mtimedb = portage.MtimeDB( + os.path.join(target_eroot, portage.CACHE_PATH, "mtimedb")) + emerge_config.running_config = emerge_config.trees[ + emerge_config.trees._running_eroot]['root_config'] + QueryCommand._db = emerge_config.trees - if result: - print(" "+yellow("*")+ " See the "+colorize("INFORM", _("CONFIGURATION FILES"))\ - + " " + _("section of the") + " " + bold("emerge")) - print(" "+yellow("*")+ " " + _("man page to learn how to update config files.")) - - -def display_news_notification(root_config, myopts): - if "news" not in root_config.settings.features: - return - portdb = root_config.trees["porttree"].dbapi - vardb = root_config.trees["vartree"].dbapi - news_counts = count_unread_news(portdb, vardb) - display_news_notifications(news_counts) + return emerge_config def getgccversion(chost): """ @@ -3089,3 +3296,772 @@ def getgccversion(chost): portage.writemsg(gcc_not_found_error, noiselevel=-1) return "[unavailable]" + +# Warn about features that may confuse users and +# lead them to report invalid bugs. +_emerge_features_warn = frozenset(['keeptemp', 'keepwork']) + +def validate_ebuild_environment(trees): + features_warn = set() + for myroot in trees: + settings = trees[myroot]["vartree"].settings + settings.validate() + features_warn.update( + _emerge_features_warn.intersection(settings.features)) + + if features_warn: + msg = "WARNING: The FEATURES variable contains one " + \ + "or more values that should be disabled under " + \ + "normal circumstances: %s" % " ".join(features_warn) + out = portage.output.EOutput() + for line in textwrap.wrap(msg, 65): + out.ewarn(line) + +def check_procfs(): + procfs_path = '/proc' + if platform.system() not in ("Linux",) or \ + os.path.ismount(procfs_path): + return os.EX_OK + msg = "It seems that %s is not mounted. You have been warned." % procfs_path + writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)), + level=logging.ERROR, noiselevel=-1) + return 1 + +def config_protect_check(trees): + for root, root_trees in trees.items(): + settings = root_trees["root_config"].settings + if not settings.get("CONFIG_PROTECT"): + msg = "!!! CONFIG_PROTECT is empty" + if settings["ROOT"] != "/": + msg += " for '%s'" % root + msg += "\n" + writemsg_level(msg, level=logging.WARN, noiselevel=-1) + +def apply_priorities(settings): + ionice(settings) + nice(settings) + +def nice(settings): + try: + os.nice(int(settings.get("PORTAGE_NICENESS", "0"))) + except (OSError, ValueError) as e: + out = portage.output.EOutput() + out.eerror("Failed to change nice value to '%s'" % \ + settings["PORTAGE_NICENESS"]) + out.eerror("%s\n" % str(e)) + +def ionice(settings): + + ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") + if ionice_cmd: + ionice_cmd = portage.util.shlex_split(ionice_cmd) + if not ionice_cmd: + return + + variables = {"PID" : str(os.getpid())} + cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + # The OS kernel probably doesn't support ionice, + # so return silently. + return + + if rval != os.EX_OK: + out = portage.output.EOutput() + out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) + out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") + +def setconfig_fallback(root_config): + setconfig = root_config.setconfig + setconfig._create_default_config() + setconfig._parse(update=True) + root_config.sets = setconfig.getSets() + +def get_missing_sets(root_config): + # emerge requires existence of "world", "selected", and "system" + missing_sets = [] + + for s in ("selected", "system", "world",): + if s not in root_config.sets: + missing_sets.append(s) + + return missing_sets + +def missing_sets_warning(root_config, missing_sets): + if len(missing_sets) > 2: + missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1]) + missing_sets_str += ', and "%s"' % missing_sets[-1] + elif len(missing_sets) == 2: + missing_sets_str = '"%s" and "%s"' % tuple(missing_sets) + else: + missing_sets_str = '"%s"' % missing_sets[-1] + msg = ["emerge: incomplete set configuration, " + \ + "missing set(s): %s" % missing_sets_str] + if root_config.sets: + msg.append(" sets defined: %s" % ", ".join(root_config.sets)) + global_config_path = portage.const.GLOBAL_CONFIG_PATH + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, + portage.const.GLOBAL_CONFIG_PATH.lstrip(os.sep)) + msg.append(" This usually means that '%s'" % \ + (os.path.join(global_config_path, "sets/portage.conf"),)) + msg.append(" is missing or corrupt.") + msg.append(" Falling back to default world and system set configuration!!!") + for line in msg: + writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1) + +def ensure_required_sets(trees): + warning_shown = False + for root_trees in trees.values(): + missing_sets = get_missing_sets(root_trees["root_config"]) + if missing_sets and not warning_shown: + warning_shown = True + missing_sets_warning(root_trees["root_config"], missing_sets) + if missing_sets: + setconfig_fallback(root_trees["root_config"]) + +def expand_set_arguments(myfiles, myaction, root_config): + retval = os.EX_OK + setconfig = root_config.setconfig + + sets = setconfig.getSets() + + # In order to know exactly which atoms/sets should be added to the + # world file, the depgraph performs set expansion later. It will get + # confused about where the atoms came from if it's not allowed to + # expand them itself. + do_not_expand = myaction is None + newargs = [] + for a in myfiles: + if a in ("system", "world"): + newargs.append(SETPREFIX+a) + else: + newargs.append(a) + myfiles = newargs + del newargs + newargs = [] + + # separators for set arguments + ARG_START = "{" + ARG_END = "}" + + for i in range(0, len(myfiles)): + if myfiles[i].startswith(SETPREFIX): + start = 0 + end = 0 + x = myfiles[i][len(SETPREFIX):] + newset = "" + while x: + start = x.find(ARG_START) + end = x.find(ARG_END) + if start > 0 and start < end: + namepart = x[:start] + argpart = x[start+1:end] + + # TODO: implement proper quoting + args = argpart.split(",") + options = {} + for a in args: + if "=" in a: + k, v = a.split("=", 1) + options[k] = v + else: + options[a] = "True" + setconfig.update(namepart, options) + newset += (x[:start-len(namepart)]+namepart) + x = x[end+len(ARG_END):] + else: + newset += x + x = "" + myfiles[i] = SETPREFIX+newset + + sets = setconfig.getSets() + + # display errors that occurred while loading the SetConfig instance + for e in setconfig.errors: + print(colorize("BAD", "Error during set creation: %s" % e)) + + unmerge_actions = ("unmerge", "prune", "clean", "depclean") + + for a in myfiles: + if a.startswith(SETPREFIX): + s = a[len(SETPREFIX):] + if s not in sets: + display_missing_pkg_set(root_config, s) + return (None, 1) + if s == "installed": + msg = ("The @installed set is deprecated and will soon be " + "removed. Please refer to bug #387059 for details.") + out = portage.output.EOutput() + for line in textwrap.wrap(msg, 50): + out.ewarn(line) + setconfig.active.append(s) + + if do_not_expand: + # Loading sets can be slow, so skip it here, in order + # to allow the depgraph to indicate progress with the + # spinner while sets are loading (bug #461412). + newargs.append(a) + continue + + try: + set_atoms = setconfig.getSetAtoms(s) + except portage.exception.PackageSetNotFound as e: + writemsg_level(("emerge: the given set '%s' " + \ + "contains a non-existent set named '%s'.\n") % \ + (s, e), level=logging.ERROR, noiselevel=-1) + if s in ('world', 'selected') and \ + SETPREFIX + e.value in sets['selected']: + writemsg_level(("Use `emerge --deselect %s%s` to " + "remove this set from world_sets.\n") % + (SETPREFIX, e,), level=logging.ERROR, + noiselevel=-1) + return (None, 1) + if myaction in unmerge_actions and \ + not sets[s].supportsOperation("unmerge"): + writemsg_level("emerge: the given set '%s' does " % s + \ + "not support unmerge operations\n", + level=logging.ERROR, noiselevel=-1) + retval = 1 + elif not set_atoms: + writemsg_level("emerge: '%s' is an empty set\n" % s, + level=logging.INFO, noiselevel=-1) + else: + newargs.extend(set_atoms) + for error_msg in sets[s].errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + else: + newargs.append(a) + return (newargs, retval) + +def repo_name_check(trees): + missing_repo_names = set() + for root_trees in trees.values(): + porttree = root_trees.get("porttree") + if porttree: + portdb = porttree.dbapi + missing_repo_names.update(portdb.getMissingRepoNames()) + if portdb.porttree_root in missing_repo_names and \ + not os.path.exists(os.path.join( + portdb.porttree_root, "profiles")): + # This is normal if $PORTDIR happens to be empty, + # so don't warn about it. + missing_repo_names.remove(portdb.porttree_root) + + # Skip warnings about missing repo_name entries for + # /usr/local/portage (see bug #248603). + try: + missing_repo_names.remove('/usr/local/portage') + except KeyError: + pass + + if missing_repo_names: + msg = [] + msg.append("WARNING: One or more repositories " + \ + "have missing repo_name entries:") + msg.append("") + for p in missing_repo_names: + msg.append("\t%s/profiles/repo_name" % (p,)) + msg.append("") + msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \ + "should be a plain text file containing a unique " + \ + "name for the repository on the first line.", 70)) + msg.append("\n") + writemsg_level("".join("%s\n" % l for l in msg), + level=logging.WARNING, noiselevel=-1) + + return bool(missing_repo_names) + +def repo_name_duplicate_check(trees): + ignored_repos = {} + for root, root_trees in trees.items(): + if 'porttree' in root_trees: + portdb = root_trees['porttree'].dbapi + if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0': + for repo_name, paths in portdb.getIgnoredRepos(): + k = (root, repo_name, portdb.getRepositoryPath(repo_name)) + ignored_repos.setdefault(k, []).extend(paths) + + if ignored_repos: + msg = [] + msg.append('WARNING: One or more repositories ' + \ + 'have been ignored due to duplicate') + msg.append(' profiles/repo_name entries:') + msg.append('') + for k in sorted(ignored_repos): + msg.append(' %s overrides' % ", ".join(k)) + for path in ignored_repos[k]: + msg.append(' %s' % (path,)) + msg.append('') + msg.extend(' ' + x for x in textwrap.wrap( + "All profiles/repo_name entries must be unique in order " + \ + "to avoid having duplicates ignored. " + \ + "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \ + "/etc/portage/make.conf if you would like to disable this warning.")) + msg.append("\n") + writemsg_level(''.join('%s\n' % l for l in msg), + level=logging.WARNING, noiselevel=-1) + + return bool(ignored_repos) + +def run_action(emerge_config): + + # skip global updates prior to sync, since it's called after sync + if emerge_config.action not in ('help', 'info', 'sync', 'version') and \ + emerge_config.opts.get('--package-moves') != 'n' and \ + _global_updates(emerge_config.trees, + emerge_config.target_config.mtimedb["updates"], + quiet=("--quiet" in emerge_config.opts)): + emerge_config.target_config.mtimedb.commit() + # Reload the whole config from scratch. + load_emerge_config(emerge_config=emerge_config) + + xterm_titles = "notitles" not in \ + emerge_config.target_config.settings.features + if xterm_titles: + xtermTitle("emerge") + + if "--digest" in emerge_config.opts: + os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest" + # Reload the whole config from scratch so that the portdbapi internal + # config is updated with new FEATURES. + load_emerge_config(emerge_config=emerge_config) + + # NOTE: adjust_configs() can map options to FEATURES, so any relevant + # options adjustments should be made prior to calling adjust_configs(). + if "--buildpkgonly" in emerge_config.opts: + emerge_config.opts["--buildpkg"] = True + + if "getbinpkg" in emerge_config.target_config.settings.features: + emerge_config.opts["--getbinpkg"] = True + + if "--getbinpkgonly" in emerge_config.opts: + emerge_config.opts["--getbinpkg"] = True + + if "--getbinpkgonly" in emerge_config.opts: + emerge_config.opts["--usepkgonly"] = True + + if "--getbinpkg" in emerge_config.opts: + emerge_config.opts["--usepkg"] = True + + if "--usepkgonly" in emerge_config.opts: + emerge_config.opts["--usepkg"] = True + + if "--buildpkgonly" in emerge_config.opts: + # --buildpkgonly will not merge anything, so + # it cancels all binary package options. + for opt in ("--getbinpkg", "--getbinpkgonly", + "--usepkg", "--usepkgonly"): + emerge_config.opts.pop(opt, None) + + adjust_configs(emerge_config.opts, emerge_config.trees) + apply_priorities(emerge_config.target_config.settings) + + for fmt in emerge_config.target_config.settings["PORTAGE_BINPKG_FORMAT"].split(): + if not fmt in portage.const.SUPPORTED_BINPKG_FORMATS: + if "--pkg-format" in emerge_config.opts: + problematic="--pkg-format" + else: + problematic="PORTAGE_BINPKG_FORMAT" + + writemsg_level(("emerge: %s is not set correctly. Format " + \ + "'%s' is not supported.\n") % (problematic, fmt), + level=logging.ERROR, noiselevel=-1) + return 1 + + if emerge_config.action == 'version': + writemsg_stdout(getportageversion( + emerge_config.target_config.settings["PORTDIR"], + None, + emerge_config.target_config.settings.profile_path, + emerge_config.target_config.settings["CHOST"], + emerge_config.target_config.trees['vartree'].dbapi) + '\n', + noiselevel=-1) + return 0 + elif emerge_config.action == 'help': + emerge_help() + return 0 + + spinner = stdout_spinner() + if "candy" in emerge_config.target_config.settings.features: + spinner.update = spinner.update_scroll + + if "--quiet" not in emerge_config.opts: + portage.deprecated_profile_check( + settings=emerge_config.target_config.settings) + repo_name_check(emerge_config.trees) + repo_name_duplicate_check(emerge_config.trees) + config_protect_check(emerge_config.trees) + check_procfs() + + for mytrees in emerge_config.trees.values(): + mydb = mytrees["porttree"].dbapi + # Freeze the portdbapi for performance (memoize all xmatch results). + mydb.freeze() + + if emerge_config.action in ('search', None) and \ + "--usepkg" in emerge_config.opts: + # Populate the bintree with current --getbinpkg setting. + # This needs to happen before expand_set_arguments(), in case + # any sets use the bintree. + mytrees["bintree"].populate( + getbinpkgs="--getbinpkg" in emerge_config.opts) + + del mytrees, mydb + + for x in emerge_config.args: + if x.endswith((".ebuild", ".tbz2")) and \ + os.path.exists(os.path.abspath(x)): + print(colorize("BAD", "\n*** emerging by path is broken " + "and may not always work!!!\n")) + break + + if emerge_config.action == "list-sets": + writemsg_stdout("".join("%s\n" % s for s in + sorted(emerge_config.target_config.sets))) + return os.EX_OK + elif emerge_config.action == "check-news": + news_counts = count_unread_news( + emerge_config.target_config.trees["porttree"].dbapi, + emerge_config.target_config.trees["vartree"].dbapi) + if any(news_counts.values()): + display_news_notifications(news_counts) + elif "--quiet" not in emerge_config.opts: + print("", colorize("GOOD", "*"), "No news items were found.") + return os.EX_OK + + ensure_required_sets(emerge_config.trees) + + if emerge_config.action is None and \ + "--resume" in emerge_config.opts and emerge_config.args: + writemsg("emerge: unexpected argument(s) for --resume: %s\n" % + " ".join(emerge_config.args), noiselevel=-1) + return 1 + + # only expand sets for actions taking package arguments + oldargs = emerge_config.args[:] + if emerge_config.action in ("clean", "config", "depclean", + "info", "prune", "unmerge", None): + newargs, retval = expand_set_arguments( + emerge_config.args, emerge_config.action, + emerge_config.target_config) + if retval != os.EX_OK: + return retval + + # Need to handle empty sets specially, otherwise emerge will react + # with the help message for empty argument lists + if oldargs and not newargs: + print("emerge: no targets left after set expansion") + return 0 + + emerge_config.args = newargs + + if "--tree" in emerge_config.opts and \ + "--columns" in emerge_config.opts: + print("emerge: can't specify both of \"--tree\" and \"--columns\".") + return 1 + + if '--emptytree' in emerge_config.opts and \ + '--noreplace' in emerge_config.opts: + writemsg_level("emerge: can't specify both of " + \ + "\"--emptytree\" and \"--noreplace\".\n", + level=logging.ERROR, noiselevel=-1) + return 1 + + if ("--quiet" in emerge_config.opts): + spinner.update = spinner.update_quiet + portage.util.noiselimit = -1 + + if "--fetch-all-uri" in emerge_config.opts: + emerge_config.opts["--fetchonly"] = True + + if "--skipfirst" in emerge_config.opts and \ + "--resume" not in emerge_config.opts: + emerge_config.opts["--resume"] = True + + # Allow -p to remove --ask + if "--pretend" in emerge_config.opts: + emerge_config.opts.pop("--ask", None) + + # forbid --ask when not in a terminal + # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway. + if ("--ask" in emerge_config.opts) and (not sys.stdin.isatty()): + portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n", + noiselevel=-1) + return 1 + + if emerge_config.target_config.settings.get("PORTAGE_DEBUG", "") == "1": + spinner.update = spinner.update_quiet + portage.util.noiselimit = 0 + if "python-trace" in emerge_config.target_config.settings.features: + portage.debug.set_trace(True) + + if not ("--quiet" in emerge_config.opts): + if '--nospinner' in emerge_config.opts or \ + emerge_config.target_config.settings.get('TERM') == 'dumb' or \ + not sys.stdout.isatty(): + spinner.update = spinner.update_basic + + if "--debug" in emerge_config.opts: + print("myaction", emerge_config.action) + print("myopts", emerge_config.opts) + + if not emerge_config.action and not emerge_config.args and \ + "--resume" not in emerge_config.opts: + emerge_help() + return 1 + + pretend = "--pretend" in emerge_config.opts + fetchonly = "--fetchonly" in emerge_config.opts or \ + "--fetch-all-uri" in emerge_config.opts + buildpkgonly = "--buildpkgonly" in emerge_config.opts + + # check if root user is the current user for the actions where emerge needs this + if portage.data.secpass < 2: + # We've already allowed "--version" and "--help" above. + if "--pretend" not in emerge_config.opts and \ + emerge_config.action not in ("search", "info"): + need_superuser = emerge_config.action in ('clean', 'depclean', + 'deselect', 'prune', 'unmerge') or not \ + (fetchonly or \ + (buildpkgonly and portage.data.secpass >= 1) or \ + emerge_config.action in ("metadata", "regen", "sync")) + if portage.data.secpass < 1 or \ + need_superuser: + if need_superuser: + access_desc = "superuser" + else: + access_desc = "portage group" + # Always show portage_group_warning() when only portage group + # access is required but the user is not in the portage group. + if "--ask" in emerge_config.opts: + writemsg_stdout("This action requires %s access...\n" % \ + (access_desc,), noiselevel=-1) + if portage.data.secpass < 1 and not need_superuser: + portage.data.portage_group_warning() + if userquery("Would you like to add --pretend to options?", + "--ask-enter-invalid" in emerge_config.opts) == "No": + return 128 + signal.SIGINT + emerge_config.opts["--pretend"] = True + emerge_config.opts.pop("--ask") + else: + sys.stderr.write(("emerge: %s access is required\n") \ + % access_desc) + if portage.data.secpass < 1 and not need_superuser: + portage.data.portage_group_warning() + return 1 + + # Disable emergelog for everything except build or unmerge operations. + # This helps minimize parallel emerge.log entries that can confuse log + # parsers like genlop. + disable_emergelog = False + for x in ("--pretend", "--fetchonly", "--fetch-all-uri"): + if x in emerge_config.opts: + disable_emergelog = True + break + if disable_emergelog: + pass + elif emerge_config.action in ("search", "info"): + disable_emergelog = True + elif portage.data.secpass < 1: + disable_emergelog = True + + import _emerge.emergelog + _emerge.emergelog._disable = disable_emergelog + + if not disable_emergelog: + emerge_log_dir = \ + emerge_config.target_config.settings.get('EMERGE_LOG_DIR') + if emerge_log_dir: + try: + # At least the parent needs to exist for the lock file. + portage.util.ensure_dirs(emerge_log_dir) + except portage.exception.PortageException as e: + writemsg_level("!!! Error creating directory for " + \ + "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \ + (emerge_log_dir, e), + noiselevel=-1, level=logging.ERROR) + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) + else: + _emerge.emergelog._emerge_log_dir = emerge_log_dir + else: + _emerge.emergelog._emerge_log_dir = os.path.join(os.sep, + portage.const.EPREFIX.lstrip(os.sep), "var", "log") + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) + + if not "--pretend" in emerge_config.opts: + time_fmt = "%b %d, %Y %H:%M:%S" + if sys.hexversion < 0x3000000: + time_fmt = portage._unicode_encode(time_fmt) + time_str = time.strftime(time_fmt, time.localtime(time.time())) + # Avoid potential UnicodeDecodeError in Python 2, since strftime + # returns bytes in Python 2, and %b may contain non-ascii chars. + time_str = _unicode_decode(time_str, + encoding=_encodings['content'], errors='replace') + emergelog(xterm_titles, "Started emerge on: %s" % time_str) + myelogstr="" + if emerge_config.opts: + opt_list = [] + for opt, arg in emerge_config.opts.items(): + if arg is True: + opt_list.append(opt) + elif isinstance(arg, list): + # arguments like --exclude that use 'append' action + for x in arg: + opt_list.append("%s=%s" % (opt, x)) + else: + opt_list.append("%s=%s" % (opt, arg)) + myelogstr=" ".join(opt_list) + if emerge_config.action: + myelogstr += " --" + emerge_config.action + if oldargs: + myelogstr += " " + " ".join(oldargs) + emergelog(xterm_titles, " *** emerge " + myelogstr) + + oldargs = None + + def emergeexitsig(signum, frame): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + portage.util.writemsg( + "\n\nExiting on signal %(signal)s\n" % {"signal":signum}) + sys.exit(128 + signum) + + signal.signal(signal.SIGTERM, emergeexitsig) + + def emergeexit(): + """This gets out final log message in before we quit.""" + if "--pretend" not in emerge_config.opts: + emergelog(xterm_titles, " *** terminating.") + if xterm_titles: + xtermTitleReset() + portage.atexit_register(emergeexit) + + if emerge_config.action in ("config", "metadata", "regen", "sync"): + if "--pretend" in emerge_config.opts: + sys.stderr.write(("emerge: The '%s' action does " + \ + "not support '--pretend'.\n") % emerge_config.action) + return 1 + + if "sync" == emerge_config.action: + return action_sync(emerge_config) + elif "metadata" == emerge_config.action: + action_metadata(emerge_config.target_config.settings, + emerge_config.target_config.trees['porttree'].dbapi, + emerge_config.opts) + elif emerge_config.action=="regen": + validate_ebuild_environment(emerge_config.trees) + return action_regen(emerge_config.target_config.settings, + emerge_config.target_config.trees['porttree'].dbapi, + emerge_config.opts.get("--jobs"), + emerge_config.opts.get("--load-average")) + # HELP action + elif "config" == emerge_config.action: + validate_ebuild_environment(emerge_config.trees) + action_config(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.opts, emerge_config.args) + + # SEARCH action + elif "search" == emerge_config.action: + validate_ebuild_environment(emerge_config.trees) + action_search(emerge_config.target_config, + emerge_config.opts, emerge_config.args, spinner) + + elif emerge_config.action in \ + ('clean', 'depclean', 'deselect', 'prune', 'unmerge'): + validate_ebuild_environment(emerge_config.trees) + rval = action_uninstall(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.target_config.mtimedb["ldpath"], + emerge_config.opts, emerge_config.action, + emerge_config.args, spinner) + if not (emerge_config.action == 'deselect' or + buildpkgonly or fetchonly or pretend): + post_emerge(emerge_config.action, emerge_config.opts, + emerge_config.args, emerge_config.target_config.root, + emerge_config.trees, emerge_config.target_config.mtimedb, rval) + return rval + + elif emerge_config.action == 'info': + + # Ensure atoms are valid before calling unmerge(). + vardb = emerge_config.target_config.trees['vartree'].dbapi + portdb = emerge_config.target_config.trees['porttree'].dbapi + bindb = emerge_config.target_config.trees['bintree'].dbapi + valid_atoms = [] + for x in emerge_config.args: + if is_valid_package_atom(x, allow_repo=True): + try: + #look at the installed files first, if there is no match + #look at the ebuilds, since EAPI 4 allows running pkg_info + #on non-installed packages + valid_atom = dep_expand(x, mydb=vardb) + if valid_atom.cp.split("/")[0] == "null": + valid_atom = dep_expand(x, mydb=portdb) + + if valid_atom.cp.split("/")[0] == "null" and \ + "--usepkg" in emerge_config.opts: + valid_atom = dep_expand(x, mydb=bindb) + + valid_atoms.append(valid_atom) + + except portage.exception.AmbiguousPackageName as e: + msg = "The short ebuild name \"" + x + \ + "\" is ambiguous. Please specify " + \ + "one of the following " + \ + "fully-qualified ebuild names instead:" + for line in textwrap.wrap(msg, 70): + writemsg_level("!!! %s\n" % (line,), + level=logging.ERROR, noiselevel=-1) + for i in e.args[0]: + writemsg_level(" %s\n" % colorize("INFORM", i), + level=logging.ERROR, noiselevel=-1) + writemsg_level("\n", level=logging.ERROR, noiselevel=-1) + return 1 + continue + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + + return action_info(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.opts, valid_atoms) + + # "update", "system", or just process files: + else: + validate_ebuild_environment(emerge_config.trees) + + for x in emerge_config.args: + if x.startswith(SETPREFIX) or \ + is_valid_package_atom(x, allow_repo=True): + continue + if x[:1] == os.sep: + continue + try: + os.lstat(x) + continue + except OSError: + pass + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + + # GLEP 42 says to display news *after* an emerge --pretend + if "--pretend" not in emerge_config.opts: + display_news_notification( + emerge_config.target_config, emerge_config.opts) + retval = action_build(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.target_config.mtimedb, + emerge_config.opts, emerge_config.action, + emerge_config.args, spinner) + post_emerge(emerge_config.action, emerge_config.opts, + emerge_config.args, emerge_config.target_config.root, + emerge_config.trees, emerge_config.target_config.mtimedb, retval) + + return retval diff --git a/portage_with_autodep/pym/_emerge/actions.pyo b/portage_with_autodep/pym/_emerge/actions.pyo Binary files differindex 4fbda01..42060b5 100644 --- a/portage_with_autodep/pym/_emerge/actions.pyo +++ b/portage_with_autodep/pym/_emerge/actions.pyo diff --git a/portage_with_autodep/pym/_emerge/clear_caches.py b/portage_with_autodep/pym/_emerge/clear_caches.py index 7b7c5ec..513df62 100644 --- a/portage_with_autodep/pym/_emerge/clear_caches.py +++ b/portage_with_autodep/pym/_emerge/clear_caches.py @@ -1,8 +1,7 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gc -from portage.util.listdir import dircache def clear_caches(trees): for d in trees.values(): @@ -15,5 +14,4 @@ def clear_caches(trees): pass else: d["vartree"].dbapi._linkmap._clear_cache() - dircache.clear() gc.collect() diff --git a/portage_with_autodep/pym/_emerge/clear_caches.pyo b/portage_with_autodep/pym/_emerge/clear_caches.pyo Binary files differindex 2e6f010..d07fbeb 100644 --- a/portage_with_autodep/pym/_emerge/clear_caches.pyo +++ b/portage_with_autodep/pym/_emerge/clear_caches.pyo diff --git a/portage_with_autodep/pym/_emerge/countdown.py b/portage_with_autodep/pym/_emerge/countdown.py index 5abdc8a..62e3c8d 100644 --- a/portage_with_autodep/pym/_emerge/countdown.py +++ b/portage_with_autodep/pym/_emerge/countdown.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -8,15 +8,15 @@ import time from portage.output import colorize -def countdown(secs=5, doing="Starting"): + +def countdown(secs=5, doing='Starting'): if secs: - print(">>> Waiting",secs,"seconds before starting...") - print(">>> (Control-C to abort)...\n"+doing+" in: ", end=' ') - ticks=list(range(secs)) - ticks.reverse() - for sec in ticks: - sys.stdout.write(colorize("UNMERGE_WARN", str(sec+1)+" ")) + print( + '>>> Waiting %s seconds before starting...\n' + '>>> (Control-C to abort)...\n' + '%s in:' % (secs, doing), end='') + for sec in range(secs, 0, -1): + sys.stdout.write(colorize('UNMERGE_WARN', ' %i' % sec)) sys.stdout.flush() time.sleep(1) print() - diff --git a/portage_with_autodep/pym/_emerge/countdown.pyo b/portage_with_autodep/pym/_emerge/countdown.pyo Binary files differindex 537dd27..37341fd 100644 --- a/portage_with_autodep/pym/_emerge/countdown.pyo +++ b/portage_with_autodep/pym/_emerge/countdown.pyo diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.py b/portage_with_autodep/pym/_emerge/create_depgraph_params.py index 8f15c68..98a7646 100644 --- a/portage_with_autodep/pym/_emerge/create_depgraph_params.py +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.py @@ -15,12 +15,22 @@ def create_depgraph_params(myopts, myaction): # complete: completely account for all known dependencies # remove: build graph for use in removing packages # rebuilt_binaries: replace installed packages with rebuilt binaries + # rebuild_if_new_slot: rebuild or reinstall packages when + # slot/sub-slot := operator dependencies can be satisfied by a newer + # slot/sub-slot, so that older packages slots will become eligible for + # removal by the --depclean action as soon as possible + # ignore_built_slot_operator_deps: ignore the slot/sub-slot := operator parts + # of dependencies that have been recorded when packages where built myparams = {"recurse" : True} bdeps = myopts.get("--with-bdeps") if bdeps is not None: myparams["bdeps"] = bdeps + ignore_built_slot_operator_deps = myopts.get("--ignore-built-slot-operator-deps") + if ignore_built_slot_operator_deps is not None: + myparams["ignore_built_slot_operator_deps"] = ignore_built_slot_operator_deps + dynamic_deps = myopts.get("--dynamic-deps") if dynamic_deps is not None: myparams["dynamic_deps"] = dynamic_deps @@ -31,6 +41,10 @@ def create_depgraph_params(myopts, myaction): myparams["selective"] = True return myparams + rebuild_if_new_slot = myopts.get('--rebuild-if-new-slot') + if rebuild_if_new_slot is not None: + myparams['rebuild_if_new_slot'] = rebuild_if_new_slot + if "--update" in myopts or \ "--newuse" in myopts or \ "--reinstall" in myopts or \ @@ -42,6 +56,11 @@ def create_depgraph_params(myopts, myaction): if deep is not None and deep != 0: myparams["deep"] = deep + complete_if_new_use = \ + myopts.get("--complete-graph-if-new-use") + if complete_if_new_use is not None: + myparams["complete_if_new_use"] = complete_if_new_use + complete_if_new_ver = \ myopts.get("--complete-graph-if-new-ver") if complete_if_new_ver is not None: diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo Binary files differindex 834580a..d9625b8 100644 --- a/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.py b/portage_with_autodep/pym/_emerge/create_world_atom.py index 35fb7c4..ac994cc 100644 --- a/portage_with_autodep/pym/_emerge/create_world_atom.py +++ b/portage_with_autodep/pym/_emerge/create_world_atom.py @@ -1,7 +1,15 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import sys + from portage.dep import _repo_separator +from portage.exception import InvalidData + +if sys.hexversion >= 0x3000000: + _unicode = str +else: + _unicode = unicode def create_world_atom(pkg, args_set, root_config): """Create a new atom for the world file if one does not exist. If the @@ -35,16 +43,15 @@ def create_world_atom(pkg, args_set, root_config): for cpv in portdb.match(cp): for repo in repos: try: - available_slots.add(portdb.aux_get(cpv, ["SLOT"], - myrepo=repo)[0]) - except KeyError: + available_slots.add(portdb._pkg_str(_unicode(cpv), repo).slot) + except (KeyError, InvalidData): pass slotted = len(available_slots) > 1 or \ (len(available_slots) == 1 and "0" not in available_slots) if not slotted: # check the vdb in case this is multislot - available_slots = set(vardb.aux_get(cpv, ["SLOT"])[0] \ + available_slots = set(vardb._pkg_str(cpv, None).slot \ for cpv in vardb.match(cp)) slotted = len(available_slots) > 1 or \ (len(available_slots) == 1 and "0" not in available_slots) @@ -83,14 +90,14 @@ def create_world_atom(pkg, args_set, root_config): matched_slots = set() if mydb is vardb: for cpv in matches: - matched_slots.add(mydb.aux_get(cpv, ["SLOT"])[0]) + matched_slots.add(mydb._pkg_str(cpv, None).slot) else: for cpv in matches: for repo in repos: try: - matched_slots.add(portdb.aux_get(cpv, ["SLOT"], - myrepo=repo)[0]) - except KeyError: + matched_slots.add( + portdb._pkg_str(_unicode(cpv), repo).slot) + except (KeyError, InvalidData): pass if len(matched_slots) == 1: diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.pyo b/portage_with_autodep/pym/_emerge/create_world_atom.pyo Binary files differindex ac3fb5d..523c5a0 100644 --- a/portage_with_autodep/pym/_emerge/create_world_atom.pyo +++ b/portage_with_autodep/pym/_emerge/create_world_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/depgraph.py b/portage_with_autodep/pym/_emerge/depgraph.py index 572cea7..763f3fd 100644 --- a/portage_with_autodep/pym/_emerge/depgraph.py +++ b/portage_with_autodep/pym/_emerge/depgraph.py @@ -1,32 +1,37 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals -import difflib import errno import io import logging import stat import sys import textwrap +import warnings from collections import deque from itertools import chain import portage from portage import os, OrderedDict from portage import _unicode_decode, _unicode_encode, _encodings -from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH +from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH, VCS_DIRS from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand +from portage.dbapi._similar_name_search import similar_name_search from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ check_required_use, human_readable_required_use, match_from_list, \ _repo_separator -from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use -from portage.exception import InvalidAtom, InvalidDependString, PortageException +from portage.dep._slot_operator import ignore_built_slot_operator_deps +from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use, \ + _get_eapi_attrs +from portage.exception import (InvalidAtom, InvalidData, InvalidDependString, + PackageNotFound, PortageException) from portage.output import colorize, create_color_func, \ darkgreen, green bad = create_color_func("BAD") +from portage.package.ebuild.config import _get_feature_flags from portage.package.ebuild.getmaskingstatus import \ _getmaskingstatus, _MaskReason from portage._sets import SETPREFIX @@ -36,13 +41,16 @@ from portage.util import cmp_sort_key, writemsg, writemsg_stdout from portage.util import ensure_dirs from portage.util import writemsg_level, write_atomic from portage.util.digraph import digraph -from portage.util.listdir import _ignorecvs_dirs +from portage.util._async.TaskScheduler import TaskScheduler +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from portage.versions import catpkgsplit from _emerge.AtomArg import AtomArg from _emerge.Blocker import Blocker from _emerge.BlockerCache import BlockerCache from _emerge.BlockerDepPriority import BlockerDepPriority +from .chk_updated_cfg_files import chk_updated_cfg_files from _emerge.countdown import countdown from _emerge.create_world_atom import create_world_atom from _emerge.Dependency import Dependency @@ -50,6 +58,7 @@ from _emerge.DependencyArg import DependencyArg from _emerge.DepPriority import DepPriority from _emerge.DepPriorityNormalRange import DepPriorityNormalRange from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange +from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.FakeVartree import FakeVartree from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge.is_valid_package_atom import insert_category_into_atom, \ @@ -73,6 +82,9 @@ from _emerge.resolver.output import Display if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class _scheduler_graph_config(object): def __init__(self, trees, pkg_cache, graph, mergelist): @@ -85,9 +97,9 @@ def _wildcard_set(atoms): pkgs = InternalPackageSet(allow_wildcard=True) for x in atoms: try: - x = Atom(x, allow_wildcard=True) + x = Atom(x, allow_wildcard=True, allow_repo=False) except portage.exception.InvalidAtom: - x = Atom("*/" + x, allow_wildcard=True) + x = Atom("*/" + x, allow_wildcard=True, allow_repo=False) pkgs.add(x) return pkgs @@ -110,6 +122,8 @@ class _frozen_depgraph_config(object): self._pkg_cache = {} self._highest_license_masked = {} dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_operator_deps = myopts.get( + "--ignore-built-slot-operator-deps", "n") == "y" for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -124,7 +138,8 @@ class _frozen_depgraph_config(object): FakeVartree(trees[myroot]["root_config"], pkg_cache=self._pkg_cache, pkg_root_config=self.roots[myroot], - dynamic_deps=dynamic_deps) + dynamic_deps=dynamic_deps, + ignore_built_slot_operator_deps=ignore_built_slot_operator_deps) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) @@ -259,13 +274,12 @@ class _rebuild_config(object): return True elif (parent.installed and root_slot not in self.reinstall_list): - inst_build_time = parent.metadata.get("BUILD_TIME") try: bin_build_time, = bindb.aux_get(parent.cpv, ["BUILD_TIME"]) except KeyError: continue - if bin_build_time != inst_build_time: + if bin_build_time != _unicode(parent.build_time): # 2) Remote binary package is valid, and local package # is not up to date. Force reinstall. reinstall = True @@ -366,12 +380,15 @@ class _dynamic_depgraph_config(object): # This use used to check if we have accounted for blockers # relevant to a package. self._traversed_pkg_deps = set() - self._slot_collision_info = {} + # This should be ordered such that the backtracker will + # attempt to solve conflicts which occurred earlier first, + # since an earlier conflict can be the cause of a conflict + # which occurs later. + self._slot_collision_info = OrderedDict() # Slot collision nodes are not allowed to block other packages since # blocker validation is only able to account for one package per slot. self._slot_collision_nodes = set() self._parent_atoms = {} - self._slot_conflict_parent_atoms = set() self._slot_conflict_handler = None self._circular_dependency_handler = None self._serialized_tasks_cache = None @@ -401,15 +418,20 @@ class _dynamic_depgraph_config(object): self._needed_license_changes = backtrack_parameters.needed_license_changes self._needed_use_config_changes = backtrack_parameters.needed_use_config_changes self._runtime_pkg_mask = backtrack_parameters.runtime_pkg_mask + self._slot_operator_replace_installed = backtrack_parameters.slot_operator_replace_installed + self._prune_rebuilds = backtrack_parameters.prune_rebuilds self._need_restart = False # For conditions that always require user intervention, such as # unsatisfied REQUIRED_USE (currently has no autounmask support). self._skip_restart = False self._backtrack_infos = {} + self._buildpkgonly_deps_unsatisfied = False self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n' self._success_without_autounmask = False self._traverse_ignored_deps = False + self._complete_mode = False + self._slot_operator_deps = {} for myroot in depgraph._frozen_config.trees: self.sets[myroot] = _depgraph_sets() @@ -432,6 +454,7 @@ class _dynamic_depgraph_config(object): self._graph_trees[myroot]["vartree"] = graph_tree self._graph_trees[myroot]["graph_db"] = graph_tree.dbapi self._graph_trees[myroot]["graph"] = self.digraph + self._graph_trees[myroot]["want_update_pkg"] = depgraph._want_update_pkg def filtered_tree(): pass filtered_tree.dbapi = _dep_check_composite_db(depgraph, myroot) @@ -458,6 +481,7 @@ class _dynamic_depgraph_config(object): self._filtered_trees[myroot]["graph"] = self.digraph self._filtered_trees[myroot]["vartree"] = \ depgraph._frozen_config.trees[myroot]["vartree"] + self._filtered_trees[myroot]["want_update_pkg"] = depgraph._want_update_pkg dbs = [] # (db, pkg_type, built, installed, db_keys) @@ -488,8 +512,6 @@ class depgraph(object): pkg_tree_map = RootConfig.pkg_tree_map - _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - def __init__(self, settings, trees, myopts, myparams, spinner, frozen_config=None, backtrack_parameters=BacktrackParameter(), allow_backtracking=False): if frozen_config is None: @@ -503,6 +525,9 @@ class depgraph(object): self._select_atoms = self._select_atoms_highest_available self._select_package = self._select_pkg_highest_available + self._event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) + def _load_vdb(self): """ Load installed package metadata if appropriate. This used to be called @@ -521,10 +546,6 @@ class depgraph(object): preload_installed_pkgs = \ "--nodeps" not in self._frozen_config.myopts - if self._frozen_config.myopts.get("--root-deps") is not None and \ - myroot != self._frozen_config.target_root: - continue - fake_vartree = self._frozen_config.trees[myroot]["vartree"] if not fake_vartree.dbapi: # This needs to be called for the first depgraph, but not for @@ -541,21 +562,156 @@ class depgraph(object): fakedb = self._dynamic_config._graph_trees[ myroot]["vartree"].dbapi - for pkg in vardb: - self._spinner_update() - if dynamic_deps: - # This causes FakeVartree to update the - # Package instance dependencies via - # PackageVirtualDbapi.aux_update() - vardb.aux_get(pkg.cpv, []) - fakedb.cpv_inject(pkg) + if not dynamic_deps: + for pkg in vardb: + fakedb.cpv_inject(pkg) + else: + max_jobs = self._frozen_config.myopts.get("--jobs") + max_load = self._frozen_config.myopts.get("--load-average") + scheduler = TaskScheduler( + self._dynamic_deps_preload(fake_vartree, fakedb), + max_jobs=max_jobs, + max_load=max_load, + event_loop=fake_vartree._portdb._event_loop) + scheduler.start() + scheduler.wait() self._dynamic_config._vdb_loaded = True + def _dynamic_deps_preload(self, fake_vartree, fakedb): + portdb = fake_vartree._portdb + for pkg in fake_vartree.dbapi: + self._spinner_update() + fakedb.cpv_inject(pkg) + ebuild_path, repo_path = \ + portdb.findname2(pkg.cpv, myrepo=pkg.repo) + if ebuild_path is None: + fake_vartree.dynamic_deps_preload(pkg, None) + continue + metadata, ebuild_hash = portdb._pull_valid_cache( + pkg.cpv, ebuild_path, repo_path) + if metadata is not None: + fake_vartree.dynamic_deps_preload(pkg, metadata) + else: + proc = EbuildMetadataPhase(cpv=pkg.cpv, + ebuild_hash=ebuild_hash, + portdb=portdb, repo_path=repo_path, + settings=portdb.doebuild_settings) + proc.addExitListener( + self._dynamic_deps_proc_exit(pkg, fake_vartree)) + yield proc + + class _dynamic_deps_proc_exit(object): + + __slots__ = ('_pkg', '_fake_vartree') + + def __init__(self, pkg, fake_vartree): + self._pkg = pkg + self._fake_vartree = fake_vartree + + def __call__(self, proc): + metadata = None + if proc.returncode == os.EX_OK: + metadata = proc.metadata + self._fake_vartree.dynamic_deps_preload(self._pkg, metadata) + def _spinner_update(self): if self._frozen_config.spinner: self._frozen_config.spinner.update() + def _compute_abi_rebuild_info(self): + """ + Fill self._forced_rebuilds with packages that cause rebuilds. + """ + + debug = "--debug" in self._frozen_config.myopts + + # Get all atoms that might have caused a forced rebuild. + atoms = {} + for s in self._dynamic_config._initial_arg_list: + if s.force_reinstall: + root = s.root_config.root + atoms.setdefault(root, set()).update(s.pset) + + if debug: + writemsg_level("forced reinstall atoms:\n", + level=logging.DEBUG, noiselevel=-1) + + for root in atoms: + writemsg_level(" root: %s\n" % root, + level=logging.DEBUG, noiselevel=-1) + for atom in atoms[root]: + writemsg_level(" atom: %s\n" % atom, + level=logging.DEBUG, noiselevel=-1) + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + # Go through all slot operator deps and check if one of these deps + # has a parent that is matched by one of the atoms from above. + forced_rebuilds = {} + for (root, slot_atom), deps in self._dynamic_config._slot_operator_deps.items(): + rebuild_atoms = atoms.get(root, set()) + + for dep in deps: + if dep.parent.installed or dep.child.installed or \ + dep.parent.slot_atom not in rebuild_atoms: + continue + + # Make sure the child's slot/subslot has changed. If it hasn't, + # then another child has forced this rebuild. + installed_pkg = self._select_pkg_from_installed(root, dep.child.slot_atom)[0] + if installed_pkg and installed_pkg.slot == dep.child.slot and \ + installed_pkg.sub_slot == dep.child.sub_slot: + continue + + # The child has forced a rebuild of the parent + forced_rebuilds.setdefault(root, {}).setdefault(dep.child, set()).add(dep.parent) + + if debug: + writemsg_level("slot operator dependencies:\n", + level=logging.DEBUG, noiselevel=-1) + + for (root, slot_atom), deps in self._dynamic_config._slot_operator_deps.items(): + writemsg_level(" (%s, %s)\n" % \ + (root, slot_atom), level=logging.DEBUG, noiselevel=-1) + for dep in deps: + writemsg_level(" parent: %s\n" % dep.parent, level=logging.DEBUG, noiselevel=-1) + writemsg_level(" child: %s (%s)\n" % (dep.child, dep.priority), level=logging.DEBUG, noiselevel=-1) + + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + + writemsg_level("forced rebuilds:\n", + level=logging.DEBUG, noiselevel=-1) + + for root in forced_rebuilds: + writemsg_level(" root: %s\n" % root, + level=logging.DEBUG, noiselevel=-1) + for child in forced_rebuilds[root]: + writemsg_level(" child: %s\n" % child, + level=logging.DEBUG, noiselevel=-1) + for parent in forced_rebuilds[root][child]: + writemsg_level(" parent: %s\n" % parent, + level=logging.DEBUG, noiselevel=-1) + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + self._forced_rebuilds = forced_rebuilds + + def _show_abi_rebuild_info(self): + + if not self._forced_rebuilds: + return + + writemsg_stdout("\nThe following packages are causing rebuilds:\n\n", noiselevel=-1) + + for root in self._forced_rebuilds: + for child in self._forced_rebuilds[root]: + writemsg_stdout(" %s causes rebuilds for:\n" % (child,), noiselevel=-1) + for parent in self._forced_rebuilds[root][child]: + writemsg_stdout(" %s\n" % (parent,), noiselevel=-1) + def _show_ignored_binaries(self): """ Show binaries that have been ignored because their USE didn't @@ -582,8 +738,7 @@ class depgraph(object): if selected_pkg.installed and \ selected_pkg.cpv == pkg.cpv and \ - selected_pkg.metadata.get('BUILD_TIME') == \ - pkg.metadata.get('BUILD_TIME'): + selected_pkg.build_time == pkg.build_time: # We don't care about ignored binaries when an # identical installed instance is selected to # fill the slot. @@ -599,11 +754,17 @@ class depgraph(object): "due to non matching USE:\n\n", noiselevel=-1) for pkg, flags in self._dynamic_config.ignored_binaries.items(): - writemsg(" =%s" % pkg.cpv, noiselevel=-1) + flag_display = [] + for flag in sorted(flags): + if flag not in pkg.use.enabled: + flag = "-" + flag + flag_display.append(flag) + flag_display = " ".join(flag_display) + # The user can paste this line into package.use + writemsg(" =%s %s" % (pkg.cpv, flag_display), noiselevel=-1) if pkg.root_config.settings["ROOT"] != "/": - writemsg(" for %s" % (pkg.root,), noiselevel=-1) - writemsg("\n use flag(s): %s\n" % ", ".join(sorted(flags)), - noiselevel=-1) + writemsg(" # for %s" % (pkg.root,), noiselevel=-1) + writemsg("\n", noiselevel=-1) msg = [ "", @@ -617,7 +778,7 @@ class depgraph(object): line = colorize("INFORM", line) writemsg(line + "\n", noiselevel=-1) - def _show_missed_update(self): + def _get_missed_updates(self): # In order to minimize noise, show only the highest # missed update from each SLOT. @@ -643,6 +804,12 @@ class depgraph(object): missed_updates[k] = (pkg, mask_type, parent_atoms) break + return missed_updates + + def _show_missed_update(self): + + missed_updates = self._get_missed_updates() + if not missed_updates: return @@ -760,7 +927,7 @@ class depgraph(object): conflict = handler.get_conflict() writemsg(conflict, noiselevel=-1) - + explanation = handler.get_explanation() if explanation: writemsg(explanation, noiselevel=-1) @@ -801,37 +968,727 @@ class depgraph(object): def _process_slot_conflicts(self): """ + If there are any slot conflicts and backtracking is enabled, + _complete_graph should complete the graph before this method + is called, so that all relevant reverse dependencies are + available for use in backtracking decisions. + """ + for (slot_atom, root), slot_nodes in \ + self._dynamic_config._slot_collision_info.items(): + self._process_slot_conflict(root, slot_atom, slot_nodes) + + def _process_slot_conflict(self, root, slot_atom, slot_nodes): + """ Process slot conflict data to identify specific atoms which lead to conflict. These atoms only match a subset of the packages that have been pulled into a given slot. """ - for (slot_atom, root), slot_nodes \ - in self._dynamic_config._slot_collision_info.items(): - all_parent_atoms = set() - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if not parent_atoms: + debug = "--debug" in self._frozen_config.myopts + + slot_parent_atoms = set() + for pkg in slot_nodes: + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if not parent_atoms: + continue + slot_parent_atoms.update(parent_atoms) + + conflict_pkgs = [] + conflict_atoms = {} + for pkg in slot_nodes: + + if self._dynamic_config._allow_backtracking and \ + pkg in self._dynamic_config._runtime_pkg_mask: + if debug: + writemsg_level( + "!!! backtracking loop detected: %s %s\n" % \ + (pkg, + self._dynamic_config._runtime_pkg_mask[pkg]), + level=logging.DEBUG, noiselevel=-1) + + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if parent_atoms is None: + parent_atoms = set() + self._dynamic_config._parent_atoms[pkg] = parent_atoms + + all_match = True + for parent_atom in slot_parent_atoms: + if parent_atom in parent_atoms: continue - all_parent_atoms.update(parent_atoms) + # Use package set for matching since it will match via + # PROVIDE when necessary, while match_from_list does not. + parent, atom = parent_atom + atom_set = InternalPackageSet( + initial_atoms=(atom,), allow_repo=True) + if atom_set.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + parent_atoms.add(parent_atom) + else: + all_match = False + conflict_atoms.setdefault(parent_atom, set()).add(pkg) - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if parent_atoms is None: - parent_atoms = set() - self._dynamic_config._parent_atoms[pkg] = parent_atoms - for parent_atom in all_parent_atoms: - if parent_atom in parent_atoms: + if not all_match: + conflict_pkgs.append(pkg) + + if conflict_pkgs and \ + self._dynamic_config._allow_backtracking and \ + not self._accept_blocker_conflicts(): + remaining = [] + for pkg in conflict_pkgs: + if self._slot_conflict_backtrack_abi(pkg, + slot_nodes, conflict_atoms): + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + config.setdefault("slot_conflict_abi", set()).add(pkg) + else: + remaining.append(pkg) + if remaining: + self._slot_confict_backtrack(root, slot_atom, + slot_parent_atoms, remaining) + + def _slot_confict_backtrack(self, root, slot_atom, + all_parents, conflict_pkgs): + + debug = "--debug" in self._frozen_config.myopts + existing_node = self._dynamic_config._slot_pkg_map[root][slot_atom] + # In order to avoid a missed update, first mask lower versions + # that conflict with higher versions (the backtracker visits + # these in reverse order). + conflict_pkgs.sort(reverse=True) + backtrack_data = [] + for to_be_masked in conflict_pkgs: + # For missed update messages, find out which + # atoms matched to_be_selected that did not + # match to_be_masked. + parent_atoms = \ + self._dynamic_config._parent_atoms.get(to_be_masked, set()) + conflict_atoms = set(parent_atom for parent_atom in all_parents \ + if parent_atom not in parent_atoms) + backtrack_data.append((to_be_masked, conflict_atoms)) + + to_be_masked = backtrack_data[-1][0] + + self._dynamic_config._backtrack_infos.setdefault( + "slot conflict", []).append(backtrack_data) + self._dynamic_config._need_restart = True + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot conflict:") + msg.append(" first package: %s" % existing_node) + msg.append(" package to mask: %s" % to_be_masked) + msg.append(" slot: %s" % slot_atom) + msg.append(" parents: %s" % ", ".join( \ + "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) + + def _slot_conflict_backtrack_abi(self, pkg, slot_nodes, conflict_atoms): + """ + If one or more conflict atoms have a slot/sub-slot dep that can be resolved + by rebuilding the parent package, then schedule the rebuild via + backtracking, and return True. Otherwise, return False. + """ + + found_update = False + for parent_atom, conflict_pkgs in conflict_atoms.items(): + parent, atom = parent_atom + if atom.slot_operator != "=" or not parent.built: + continue + + if pkg not in conflict_pkgs: + continue + + for other_pkg in slot_nodes: + if other_pkg in conflict_pkgs: + continue + + dep = Dependency(atom=atom, child=other_pkg, + parent=parent, root=pkg.root) + + new_dep = \ + self._slot_operator_update_probe_slot_conflict(dep) + if new_dep is not None: + self._slot_operator_update_backtrack(dep, + new_dep=new_dep) + found_update = True + + return found_update + + def _slot_change_probe(self, dep): + """ + @rtype: bool + @return: True if dep.child should be rebuilt due to a change + in sub-slot (without revbump, as in bug #456208). + """ + if not (isinstance(dep.parent, Package) and \ + not dep.parent.built and dep.child.built): + return None + + root_config = self._frozen_config.roots[dep.root] + matches = [] + try: + matches.append(self._pkg(dep.child.cpv, "ebuild", + root_config, myrepo=dep.child.repo)) + except PackageNotFound: + pass + + for unbuilt_child in chain(matches, + self._iter_match_pkgs(root_config, "ebuild", + Atom("=%s" % (dep.child.cpv,)))): + if unbuilt_child in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage( + unbuilt_child, + modified_use=self._pkg_use_enabled(unbuilt_child)): + continue + if not self._pkg_visibility_check(unbuilt_child): + continue + break + else: + return None + + if unbuilt_child.slot == dep.child.slot and \ + unbuilt_child.sub_slot == dep.child.sub_slot: + return None + + return unbuilt_child + + def _slot_change_backtrack(self, dep, new_child_slot): + child = dep.child + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot/sub-slot change:") + msg.append(" child package: %s" % child) + msg.append(" child slot: %s/%s" % + (child.slot, child.sub_slot)) + msg.append(" new child: %s" % new_child_slot) + msg.append(" new child slot: %s/%s" % + (new_child_slot.slot, new_child_slot.sub_slot)) + msg.append(" parent package: %s" % dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + masks = {} + if not child.installed: + masks.setdefault(dep.child, {})["slot_operator_mask_built"] = None + if masks: + config.setdefault("slot_operator_mask_built", {}).update(masks) + + # trigger replacement of installed packages if necessary + reinstalls = set() + if child.installed: + replacement_atom = self._replace_installed_atom(child) + if replacement_atom is not None: + reinstalls.add((child.root, replacement_atom)) + if reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_operator_update_backtrack(self, dep, new_child_slot=None, + new_dep=None): + if new_child_slot is None: + child = dep.child + else: + child = new_child_slot + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to missed slot abi update:") + msg.append(" child package: %s" % child) + if new_child_slot is not None: + msg.append(" new child slot package: %s" % new_child_slot) + msg.append(" parent package: %s" % dep.parent) + if new_dep is not None: + msg.append(" new parent pkg: %s" % new_dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + abi_masks = {} + if new_child_slot is None: + if not child.installed: + abi_masks.setdefault(child, {})["slot_operator_mask_built"] = None + if not dep.parent.installed: + abi_masks.setdefault(dep.parent, {})["slot_operator_mask_built"] = None + if abi_masks: + config.setdefault("slot_operator_mask_built", {}).update(abi_masks) + + # trigger replacement of installed packages if necessary + abi_reinstalls = set() + if dep.parent.installed: + if new_dep is not None: + replacement_atom = new_dep.parent.slot_atom + else: + replacement_atom = self._replace_installed_atom(dep.parent) + if replacement_atom is not None: + abi_reinstalls.add((dep.parent.root, replacement_atom)) + if new_child_slot is None and child.installed: + replacement_atom = self._replace_installed_atom(child) + if replacement_atom is not None: + abi_reinstalls.add((child.root, replacement_atom)) + if abi_reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(abi_reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_operator_update_probe_slot_conflict(self, dep): + new_dep = self._slot_operator_update_probe(dep, slot_conflict=True) + + if new_dep is not None: + return new_dep + + if self._dynamic_config._autounmask is True: + + for autounmask_level in self._autounmask_levels(): + + new_dep = self._slot_operator_update_probe(dep, + slot_conflict=True, autounmask_level=autounmask_level) + + if new_dep is not None: + return new_dep + + return None + + def _slot_operator_update_probe(self, dep, new_child_slot=False, + slot_conflict=False, autounmask_level=None): + """ + slot/sub-slot := operators tend to prevent updates from getting pulled in, + since installed packages pull in packages with the slot/sub-slot that they + were built against. Detect this case so that we can schedule rebuilds + and reinstalls when appropriate. + NOTE: This function only searches for updates that involve upgrades + to higher versions, since the logic required to detect when a + downgrade would be desirable is not implemented. + """ + + if dep.child.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.child, + modified_use=self._pkg_use_enabled(dep.child)): + return None + + if dep.parent.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.parent, + modified_use=self._pkg_use_enabled(dep.parent)): + return None + + debug = "--debug" in self._frozen_config.myopts + selective = "selective" in self._dynamic_config.myparams + want_downgrade = None + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom, autounmask_level=autounmask_level): + + selected_atoms = None + + atoms = set() + invalid_metadata = False + for dep_key in ("DEPEND", "HDEPEND", "RDEPEND", "PDEPEND"): + dep_string = replacement_parent._metadata[dep_key] + if not dep_string: + continue + + try: + dep_string = portage.dep.use_reduce(dep_string, + uselist=self._pkg_use_enabled(replacement_parent), + is_valid_flag=replacement_parent.iuse.is_valid_flag, + flat=True, token_class=Atom, + eapi=replacement_parent.eapi) + except portage.exception.InvalidDependString: + invalid_metadata = True + break + + atoms.update(token for token in dep_string if isinstance(token, Atom)) + + if invalid_metadata: + continue + + # List of list of child,atom pairs for each atom. + replacement_candidates = [] + # Set of all packages all atoms can agree on. + all_candidate_pkgs = None + + for atom in atoms: + if atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + unevaluated_atom = atom.unevaluated_atom + atom = atom.without_use + + if replacement_parent.built and \ + portage.dep._match_slot(atom, dep.child): + # Our selected replacement_parent appears to be built + # for the existing child selection. So, discard this + # parent and search for another. + break + + candidate_pkg_atoms = [] + candidate_pkgs = [] + for pkg in self._iter_similar_available( + dep.child, atom): + if pkg.slot == dep.child.slot and \ + pkg.sub_slot == dep.child.sub_slot: + # If slot/sub-slot is identical, then there's + # no point in updating. continue - # Use package set for matching since it will match via - # PROVIDE when necessary, while match_from_list does not. - parent, atom = parent_atom - atom_set = InternalPackageSet( - initial_atoms=(atom,), allow_repo=True) - if atom_set.findAtomForPackage(pkg, modified_use=self._pkg_use_enabled(pkg)): - parent_atoms.add(parent_atom) + if new_child_slot: + if pkg.slot == dep.child.slot: + continue + if pkg < dep.child: + # the new slot only matters if the + # package version is higher + continue else: - self._dynamic_config._slot_conflict_parent_atoms.add(parent_atom) + if pkg.slot != dep.child.slot: + continue + if pkg < dep.child: + if want_downgrade is None: + want_downgrade = self._downgrade_probe(dep.child) + # be careful not to trigger a rebuild when + # the only version available with a + # different slot_operator is an older version + if not want_downgrade: + continue + + insignificant = False + if not slot_conflict and \ + selective and \ + dep.parent.installed and \ + dep.child.installed and \ + dep.parent.cpv == replacement_parent.cpv and \ + dep.child.cpv == pkg.cpv: + # Then can happen if the child's sub-slot changed + # without a revision bump. The sub-slot change is + # considered insignificant until one of its parent + # packages needs to be rebuilt (which may trigger a + # slot conflict). + insignificant = True + + if not insignificant: + # Evaluate USE conditionals and || deps, in order + # to see if this atom is really desirable, since + # otherwise we may trigger an undesirable rebuild + # as in bug #460304. + if selected_atoms is None: + selected_atoms = self._select_atoms_probe( + dep.child.root, replacement_parent) + if unevaluated_atom not in selected_atoms: + continue + + if not insignificant: + candidate_pkg_atoms.append((pkg, unevaluated_atom)) + candidate_pkgs.append(pkg) + + replacement_candidates.append(candidate_pkg_atoms) + if all_candidate_pkgs is None: + all_candidate_pkgs = set(candidate_pkgs) + else: + all_candidate_pkgs.intersection_update(candidate_pkgs) + + if not all_candidate_pkgs: + # If the atoms that connect parent and child can't agree on + # any replacement child, we can't do anything. + continue + + # Now select one of the pkgs as replacement. This is as easy as + # selecting the highest version. + # The more complicated part is to choose an atom for the + # new Dependency object. Choose the one which ranked the selected + # parent highest. + selected = None + for candidate_pkg_atoms in replacement_candidates: + for i, (pkg, atom) in enumerate(candidate_pkg_atoms): + if pkg not in all_candidate_pkgs: + continue + if selected is None or \ + selected[0] < pkg or \ + (selected[0] is pkg and i < selected[2]): + selected = (pkg, atom, i) + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % selected[0]) + msg.append(" new parent package: %s" % replacement_parent) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return Dependency(parent=replacement_parent, + child=selected[0], atom=selected[1]) + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % None) + msg.append(" new parent package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return None + + def _slot_operator_unsatisfied_probe(self, dep): + + if dep.parent.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.parent, + modified_use=self._pkg_use_enabled(dep.parent)): + return False + + debug = "--debug" in self._frozen_config.myopts + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom): + + for atom in replacement_parent.validated_atoms: + if not atom.slot_operator == "=" or \ + atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + atom = atom.without_use + + pkg, existing_node = self._select_package(dep.root, atom, + onlydeps=dep.onlydeps) + + if pkg is not None: + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_unsatisfied_probe:") + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" existing parent atom: %s" % dep.atom) + msg.append(" new parent package: %s" % replacement_parent) + msg.append(" new child package: %s" % pkg) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return True + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_unsatisfied_probe:") + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" existing parent atom: %s" % dep.atom) + msg.append(" new parent package: %s" % None) + msg.append(" new child package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return False + + def _slot_operator_unsatisfied_backtrack(self, dep): + + parent = dep.parent + + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to unsatisfied " + "built slot-operator dep:") + msg.append(" parent package: %s" % parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + masks = {} + if not parent.installed: + masks.setdefault(parent, {})["slot_operator_mask_built"] = None + if masks: + config.setdefault("slot_operator_mask_built", {}).update(masks) + + # trigger replacement of installed packages if necessary + reinstalls = set() + if parent.installed: + replacement_atom = self._replace_installed_atom(parent) + if replacement_atom is not None: + reinstalls.add((parent.root, replacement_atom)) + if reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(reinstalls) + + self._dynamic_config._need_restart = True + + def _downgrade_probe(self, pkg): + """ + Detect cases where a downgrade of the given package is considered + desirable due to the current version being masked or unavailable. + """ + available_pkg = None + for available_pkg in self._iter_similar_available(pkg, + pkg.slot_atom): + if available_pkg >= pkg: + # There's an available package of the same or higher + # version, so downgrade seems undesirable. + return False + + return available_pkg is not None + + def _select_atoms_probe(self, root, pkg): + selected_atoms = [] + use = self._pkg_use_enabled(pkg) + for k in pkg._dep_keys: + v = pkg._metadata.get(k) + if not v: + continue + selected_atoms.extend(self._select_atoms( + root, v, myuse=use, parent=pkg)[pkg]) + return frozenset(x.unevaluated_atom for + x in selected_atoms) + + def _iter_similar_available(self, graph_pkg, atom, autounmask_level=None): + """ + Given a package that's in the graph, do a rough check to + see if a similar package is available to install. The given + graph_pkg itself may be yielded only if it's not installed. + """ + + usepkgonly = "--usepkgonly" in self._frozen_config.myopts + useoldpkg_atoms = self._frozen_config.useoldpkg_atoms + use_ebuild_visibility = self._frozen_config.myopts.get( + '--use-ebuild-visibility', 'n') != 'n' + + for pkg in self._iter_match_pkgs_any( + graph_pkg.root_config, atom): + if pkg.cp != graph_pkg.cp: + # discard old-style virtual match + continue + if pkg.installed: + continue + if pkg in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + continue + if pkg.built: + if self._equiv_binary_installed(pkg): + continue + if not (not use_ebuild_visibility and + (usepkgonly or useoldpkg_atoms.findAtomForPackage( + pkg, modified_use=self._pkg_use_enabled(pkg)))) and \ + not self._equiv_ebuild_visible(pkg, + autounmask_level=autounmask_level): + continue + if not self._pkg_visibility_check(pkg, + autounmask_level=autounmask_level): + continue + yield pkg + + def _replace_installed_atom(self, inst_pkg): + """ + Given an installed package, generate an atom suitable for + slot_operator_replace_installed backtracking info. The replacement + SLOT may differ from the installed SLOT, so first search by cpv. + """ + built_pkgs = [] + for pkg in self._iter_similar_available(inst_pkg, + Atom("=%s" % inst_pkg.cpv)): + if not pkg.built: + return pkg.slot_atom + elif not pkg.installed: + # avoid using SLOT from a built instance + built_pkgs.append(pkg) + + for pkg in self._iter_similar_available(inst_pkg, inst_pkg.slot_atom): + if not pkg.built: + return pkg.slot_atom + elif not pkg.installed: + # avoid using SLOT from a built instance + built_pkgs.append(pkg) + + if built_pkgs: + best_version = None + for pkg in built_pkgs: + if best_version is None or pkg > best_version: + best_version = pkg + return best_version.slot_atom + + return None + + def _slot_operator_trigger_reinstalls(self): + """ + Search for packages with slot-operator deps on older slots, and schedule + rebuilds if they can link to a newer slot that's in the graph. + """ + + rebuild_if_new_slot = self._dynamic_config.myparams.get( + "rebuild_if_new_slot", "y") == "y" + + for slot_key, slot_info in self._dynamic_config._slot_operator_deps.items(): + + for dep in slot_info: + + atom = dep.atom + if atom.slot_operator is None: + continue + + if not atom.slot_operator_built: + new_child_slot = self._slot_change_probe(dep) + if new_child_slot is not None: + self._slot_change_backtrack(dep, new_child_slot) + continue + + if not (dep.parent and + isinstance(dep.parent, Package) and dep.parent.built): + continue + + # Check for slot update first, since we don't want to + # trigger reinstall of the child package when a newer + # slot will be used instead. + if rebuild_if_new_slot: + new_dep = self._slot_operator_update_probe(dep, + new_child_slot=True) + if new_dep is not None: + self._slot_operator_update_backtrack(dep, + new_child_slot=new_dep.child) + + if dep.want_update: + if self._slot_operator_update_probe(dep): + self._slot_operator_update_backtrack(dep) def _reinstall_for_flags(self, pkg, forced_flags, orig_use, orig_iuse, cur_use, cur_iuse): @@ -845,18 +1702,22 @@ class depgraph(object): in ("y", "auto")) newuse = "--newuse" in self._frozen_config.myopts changed_use = "changed-use" == self._frozen_config.myopts.get("--reinstall") + feature_flags = _get_feature_flags( + _get_eapi_attrs(pkg.eapi)) if newuse or (binpkg_respect_use and not changed_use): flags = set(orig_iuse.symmetric_difference( cur_iuse).difference(forced_flags)) flags.update(orig_iuse.intersection(orig_use).symmetric_difference( cur_iuse.intersection(cur_use))) + flags.difference_update(feature_flags) if flags: return flags elif changed_use or binpkg_respect_use: - flags = orig_iuse.intersection(orig_use).symmetric_difference( - cur_iuse.intersection(cur_use)) + flags = set(orig_iuse.intersection(orig_use).symmetric_difference( + cur_iuse.intersection(cur_use))) + flags.difference_update(feature_flags) if flags: return flags return None @@ -954,7 +1815,7 @@ class depgraph(object): # The blocker applies to the root where # the parent is or will be installed. blocker = Blocker(atom=dep.atom, - eapi=dep.parent.metadata["EAPI"], + eapi=dep.parent.eapi, priority=dep.priority, root=dep.parent.root) self._dynamic_config._blocker_parents.add(blocker, dep.parent) return 1 @@ -991,9 +1852,17 @@ class depgraph(object): (dep.parent, self._dynamic_config._runtime_pkg_mask[ dep.parent]), noiselevel=-1) - elif not self.need_restart(): + elif dep.atom.slot_operator_built and \ + self._slot_operator_unsatisfied_probe(dep): + self._slot_operator_unsatisfied_backtrack(dep) + return 1 + else: # Do not backtrack if only USE have to be changed in - # order to satisfy the dependency. + # order to satisfy the dependency. Note that when + # want_restart_for_use_change sets the need_restart + # flag, it causes _select_pkg_highest_available to + # return None, and eventually we come through here + # and skip the "missing dependency" backtracking path. dep_pkg, existing_node = \ self._select_package(dep.root, dep.atom.without_use, onlydeps=dep.onlydeps) @@ -1100,12 +1969,13 @@ class depgraph(object): # package selection, since we want to prompt the user # for USE adjustment rather than have REQUIRED_USE # affect package selection and || dep choices. - if not pkg.built and pkg.metadata.get("REQUIRED_USE") and \ - eapi_has_required_use(pkg.metadata["EAPI"]): + if not pkg.built and pkg._metadata.get("REQUIRED_USE") and \ + eapi_has_required_use(pkg.eapi): required_use_is_sat = check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag) + pkg.iuse.is_valid_flag, + eapi=pkg.eapi) if not required_use_is_sat: if dep.atom is not None and dep.parent is not None: self._add_parent_atom(pkg, (dep.parent, dep.atom)) @@ -1132,140 +2002,27 @@ class depgraph(object): if existing_node: if existing_node_matches: # The existing node can be reused. - if arg_atoms: - for parent_atom in arg_atoms: - parent, atom = parent_atom - self._dynamic_config.digraph.add(existing_node, parent, - priority=priority) - self._add_parent_atom(existing_node, parent_atom) - # If a direct circular dependency is not an unsatisfied - # buildtime dependency then drop it here since otherwise - # it can skew the merge order calculation in an unwanted - # way. - if existing_node != myparent or \ - (priority.buildtime and not priority.satisfied): - self._dynamic_config.digraph.addnode(existing_node, myparent, - priority=priority) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(existing_node, - (dep.parent, dep.atom)) - return 1 - else: - # A slot conflict has occurred. - # The existing node should not already be in - # runtime_pkg_mask, since that would trigger an - # infinite backtracking loop. - if self._dynamic_config._allow_backtracking and \ - existing_node in \ - self._dynamic_config._runtime_pkg_mask: - if "--debug" in self._frozen_config.myopts: - writemsg( - "!!! backtracking loop detected: %s %s\n" % \ - (existing_node, - self._dynamic_config._runtime_pkg_mask[ - existing_node]), noiselevel=-1) - elif self._dynamic_config._allow_backtracking and \ - not self._accept_blocker_conflicts() and \ - not self.need_restart(): - - self._add_slot_conflict(pkg) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(pkg, (dep.parent, dep.atom)) - - if arg_atoms: - for parent_atom in arg_atoms: - parent, atom = parent_atom - self._add_parent_atom(pkg, parent_atom) - self._process_slot_conflicts() - - backtrack_data = [] - fallback_data = [] - all_parents = set() - # The ordering of backtrack_data can make - # a difference here, because both mask actions may lead - # to valid, but different, solutions and the one with - # 'existing_node' masked is usually the better one. Because - # of that, we choose an order such that - # the backtracker will first explore the choice with - # existing_node masked. The backtracker reverses the - # order, so the order it uses is the reverse of the - # order shown here. See bug #339606. - for to_be_selected, to_be_masked in (existing_node, pkg), (pkg, existing_node): - # For missed update messages, find out which - # atoms matched to_be_selected that did not - # match to_be_masked. - parent_atoms = \ - self._dynamic_config._parent_atoms.get(to_be_selected, set()) - if parent_atoms: - conflict_atoms = self._dynamic_config._slot_conflict_parent_atoms.intersection(parent_atoms) - if conflict_atoms: - parent_atoms = conflict_atoms - - all_parents.update(parent_atoms) - - all_match = True - for parent, atom in parent_atoms: - i = InternalPackageSet(initial_atoms=(atom,), - allow_repo=True) - if not i.findAtomForPackage(to_be_masked): - all_match = False - break - - fallback_data.append((to_be_masked, parent_atoms)) - - if all_match: - # 'to_be_masked' does not violate any parent atom, which means - # there is no point in masking it. - pass - else: - backtrack_data.append((to_be_masked, parent_atoms)) - - if not backtrack_data: - # This shouldn't happen, but fall back to the old - # behavior if this gets triggered somehow. - backtrack_data = fallback_data - - if len(backtrack_data) > 1: - # NOTE: Generally, we prefer to mask the higher - # version since this solves common cases in which a - # lower version is needed so that all dependencies - # will be satisfied (bug #337178). However, if - # existing_node happens to be installed then we - # mask that since this is a common case that is - # triggered when --update is not enabled. - if existing_node.installed: - pass - elif pkg > existing_node: - backtrack_data.reverse() - - to_be_masked = backtrack_data[-1][0] + if pkg != existing_node: + pkg = existing_node + previously_added = True + try: + arg_atoms = list(self._iter_atoms_for_pkg(pkg)) + except InvalidDependString as e: + if not pkg.installed: + # should have been masked before + # it was selected + raise - self._dynamic_config._backtrack_infos["slot conflict"] = backtrack_data - self._dynamic_config._need_restart = True - if "--debug" in self._frozen_config.myopts: - msg = [] - msg.append("") - msg.append("") - msg.append("backtracking due to slot conflict:") - if backtrack_data is fallback_data: - msg.append("!!! backtrack_data fallback") - msg.append(" first package: %s" % existing_node) - msg.append(" second package: %s" % pkg) - msg.append(" package to mask: %s" % to_be_masked) - msg.append(" slot: %s" % pkg.slot_atom) - msg.append(" parents: %s" % ", ".join( \ - "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) - msg.append("") - writemsg_level("".join("%s\n" % l for l in msg), - noiselevel=-1, level=logging.DEBUG) - return 0 + if debug: + writemsg_level( + "%s%s %s\n" % ("Re-used Child:".ljust(15), + pkg, pkg_use_display(pkg, + self._frozen_config.myopts, + modified_use=self._pkg_use_enabled(pkg))), + level=logging.DEBUG, noiselevel=-1) - # A slot collision has occurred. Sometimes this coincides - # with unresolvable blockers, so the slot collision will be - # shown later if there are no unresolvable blockers. + else: self._add_slot_conflict(pkg) - slot_collision = True - if debug: writemsg_level( "%s%s %s\n" % ("Slot Conflict:".ljust(15), @@ -1274,6 +2031,8 @@ class depgraph(object): modified_use=self._pkg_use_enabled(existing_node))), level=logging.DEBUG, noiselevel=-1) + slot_collision = True + if slot_collision: # Now add this node to the graph so that self.display() # can show use flags and --tree portage.output. This node is @@ -1298,11 +2057,11 @@ class depgraph(object): # doesn't already. Any pre-existing providers will be preferred # over this one. try: - pkgsettings.setinst(pkg.cpv, pkg.metadata) + pkgsettings.setinst(pkg.cpv, pkg._metadata) # For consistency, also update the global virtuals. settings = self._frozen_config.roots[pkg.root].settings settings.unlock() - settings.setinst(pkg.cpv, pkg.metadata) + settings.setinst(pkg.cpv, pkg._metadata) settings.lock() except portage.exception.InvalidDependString: if not pkg.installed: @@ -1312,12 +2071,19 @@ class depgraph(object): if arg_atoms: self._dynamic_config._set_nodes.add(pkg) - # Do this even when addme is False (--onlydeps) so that the + # Do this even for onlydeps, so that the # parent/child relationship is always known in case # self._show_slot_collision_notice() needs to be called later. - self._dynamic_config.digraph.add(pkg, myparent, priority=priority) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(pkg, (dep.parent, dep.atom)) + # If a direct circular dependency is not an unsatisfied + # buildtime dependency then drop it here since otherwise + # it can skew the merge order calculation in an unwanted + # way. + if pkg != dep.parent or \ + (priority.buildtime and not priority.satisfied): + self._dynamic_config.digraph.add(pkg, + dep.parent, priority=priority) + if dep.atom is not None and dep.parent is not None: + self._add_parent_atom(pkg, (dep.parent, dep.atom)) if arg_atoms: for parent_atom in arg_atoms: @@ -1330,10 +2096,27 @@ class depgraph(object): # Installing package A, we need to make sure package A's deps are met. # emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec # If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. - if arg_atoms: - depth = 0 + if arg_atoms and depth > 0: + for parent, atom in arg_atoms: + if parent.reset_depth: + depth = 0 + break + + if previously_added and pkg.depth is not None: + depth = min(pkg.depth, depth) pkg.depth = depth deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + dep.want_update = (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + dep.child = pkg + if (not pkg.onlydeps and + dep.atom and dep.atom.slot_operator is not None): + self._add_slot_operator_dep(dep) + recurse = deep is True or depth + 1 <= deep dep_stack = self._dynamic_config._dep_stack if "recurse" not in self._dynamic_config.myparams: @@ -1365,6 +2148,14 @@ class depgraph(object): self._dynamic_config._parent_atoms[pkg] = parent_atoms parent_atoms.add(parent_atom) + def _add_slot_operator_dep(self, dep): + slot_key = (dep.root, dep.child.slot_atom) + slot_info = self._dynamic_config._slot_operator_deps.get(slot_key) + if slot_info is None: + slot_info = [] + self._dynamic_config._slot_operator_deps[slot_key] = slot_info + slot_info.append(dep) + def _add_slot_conflict(self, pkg): self._dynamic_config._slot_collision_nodes.add(pkg) slot_key = (pkg.slot_atom, pkg.root) @@ -1378,12 +2169,12 @@ class depgraph(object): def _add_pkg_deps(self, pkg, allow_unsatisfied=False): myroot = pkg.root - metadata = pkg.metadata + metadata = pkg._metadata removal_action = "remove" in self._dynamic_config.myparams + eapi_attrs = _get_eapi_attrs(pkg.eapi) edepend={} - depkeys = ["DEPEND","RDEPEND","PDEPEND"] - for k in depkeys: + for k in Package._dep_keys: edepend[k] = metadata[k] if not pkg.built and \ @@ -1410,31 +2201,44 @@ class depgraph(object): # Removal actions never traverse ignored buildtime # dependencies, so it's safe to discard them early. edepend["DEPEND"] = "" + edepend["HDEPEND"] = "" ignore_build_time_deps = True + ignore_depend_deps = ignore_build_time_deps + ignore_hdepend_deps = ignore_build_time_deps + if removal_action: depend_root = myroot else: - depend_root = self._frozen_config._running_root.root - root_deps = self._frozen_config.myopts.get("--root-deps") - if root_deps is not None: - if root_deps is True: - depend_root = myroot - elif root_deps == "rdeps": - ignore_build_time_deps = True + if eapi_attrs.hdepend: + depend_root = myroot + else: + depend_root = self._frozen_config._running_root.root + root_deps = self._frozen_config.myopts.get("--root-deps") + if root_deps is not None: + if root_deps is True: + depend_root = myroot + elif root_deps == "rdeps": + ignore_depend_deps = True # If rebuild mode is not enabled, it's safe to discard ignored # build-time dependencies. If you want these deps to be traversed # in "complete" mode then you need to specify --with-bdeps=y. - if ignore_build_time_deps and \ - not self._rebuild.rebuild: - edepend["DEPEND"] = "" + if not self._rebuild.rebuild: + if ignore_depend_deps: + edepend["DEPEND"] = "" + if ignore_hdepend_deps: + edepend["HDEPEND"] = "" deps = ( (depend_root, edepend["DEPEND"], self._priority(buildtime=True, - optional=(pkg.built or ignore_build_time_deps), - ignored=ignore_build_time_deps)), + optional=(pkg.built or ignore_depend_deps), + ignored=ignore_depend_deps)), + (self._frozen_config._running_root.root, edepend["HDEPEND"], + self._priority(buildtime=True, + optional=(pkg.built or ignore_hdepend_deps), + ignored=ignore_hdepend_deps)), (myroot, edepend["RDEPEND"], self._priority(runtime=True)), (myroot, edepend["PDEPEND"], @@ -1456,7 +2260,10 @@ class depgraph(object): try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg), is_valid_flag=pkg.iuse.is_valid_flag) + uselist=self._pkg_use_enabled(pkg), + is_valid_flag=pkg.iuse.is_valid_flag, + opconvert=True, token_class=Atom, + eapi=pkg.eapi) except portage.exception.InvalidDependString as e: if not pkg.installed: # should have been masked before it was selected @@ -1468,7 +2275,9 @@ class depgraph(object): # practical to ignore this issue for installed packages. try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg)) + uselist=self._pkg_use_enabled(pkg), + opconvert=True, token_class=Atom, + eapi=pkg.eapi) except portage.exception.InvalidDependString as e: self._dynamic_config._masked_installed.add(pkg) del e @@ -1489,9 +2298,6 @@ class depgraph(object): if not dep_string: continue - dep_string = portage.dep.paren_enclose(dep_string, - unevaluated_atom=True) - if not self._add_pkg_dep_string( pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): @@ -1514,6 +2320,37 @@ class depgraph(object): finally: self._dynamic_config._autounmask = _autounmask_backup + def _ignore_dependency(self, atom, pkg, child, dep, mypriority, recurse_satisfied): + """ + In some cases, dep_check will return deps that shouldn't + be processed any further, so they are identified and + discarded here. Try to discard as few as possible since + discarded dependencies reduce the amount of information + available for optimization of merge order. + Don't ignore dependencies if pkg has a slot operator dependency on the child + and the child has changed slot/sub_slot. + """ + if not mypriority.satisfied: + return False + slot_operator_rebuild = False + if atom.slot_operator == '=' and \ + (pkg.root, pkg.slot_atom) in self._dynamic_config._slot_operator_replace_installed and \ + mypriority.satisfied is not child and \ + mypriority.satisfied.installed and \ + child and \ + not child.installed and \ + (child.slot != mypriority.satisfied.slot or child.sub_slot != mypriority.satisfied.sub_slot): + slot_operator_rebuild = True + + return not atom.blocker and \ + not recurse_satisfied and \ + mypriority.satisfied.visible and \ + dep.child is not None and \ + not dep.child.installed and \ + self._dynamic_config._slot_pkg_map[dep.child.root].get( + dep.child.slot_atom) is None and \ + not slot_operator_rebuild + def _wrapped_add_pkg_dep_string(self, pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): depth = pkg.depth + 1 @@ -1525,7 +2362,9 @@ class depgraph(object): if debug: writemsg_level("\nParent: %s\n" % (pkg,), noiselevel=-1, level=logging.DEBUG) - writemsg_level("Depstring: %s\n" % (dep_string,), + dep_repr = portage.dep.paren_enclose(dep_string, + unevaluated_atom=True, opconvert=True) + writemsg_level("Depstring: %s\n" % (dep_repr,), noiselevel=-1, level=logging.DEBUG) writemsg_level("Priority: %s\n" % (dep_priority,), noiselevel=-1, level=logging.DEBUG) @@ -1570,6 +2409,13 @@ class depgraph(object): mypriority = dep_priority.copy() if not atom.blocker: + + if atom.slot_operator == "=": + if mypriority.buildtime: + mypriority.buildtime_slot_op = True + if mypriority.runtime: + mypriority.runtime_slot_op = True + inst_pkgs = [inst_pkg for inst_pkg in reversed(vardb.match_pkgs(atom)) if not reinstall_atoms.findAtomForPackage(inst_pkg, @@ -1589,30 +2435,18 @@ class depgraph(object): priority=mypriority, root=dep_root) # In some cases, dep_check will return deps that shouldn't - # be proccessed any further, so they are identified and + # be processed any further, so they are identified and # discarded here. Try to discard as few as possible since # discarded dependencies reduce the amount of information # available for optimization of merge order. ignored = False - if not atom.blocker and \ - not recurse_satisfied and \ - mypriority.satisfied and \ - mypriority.satisfied.visible and \ - dep.child is not None and \ - not dep.child.installed and \ - self._dynamic_config._slot_pkg_map[dep.child.root].get( - dep.child.slot_atom) is None: + if self._ignore_dependency(atom, pkg, child, dep, mypriority, recurse_satisfied): myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - # This shouldn't happen since the package - # should have been masked. - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: # Existing child selection may not be valid unless @@ -1709,23 +2543,13 @@ class depgraph(object): collapsed_parent=pkg, collapsed_priority=dep_priority) ignored = False - if not atom.blocker and \ - not recurse_satisfied and \ - mypriority.satisfied and \ - mypriority.satisfied.visible and \ - dep.child is not None and \ - not dep.child.installed and \ - self._dynamic_config._slot_pkg_map[dep.child.root].get( - dep.child.slot_atom) is None: + if self._ignore_dependency(atom, pkg, child, dep, mypriority, recurse_satisfied): myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: ignored = True @@ -1767,7 +2591,7 @@ class depgraph(object): yield (atom, None) continue dep_pkg, existing_node = self._select_package( - root_config.root, atom) + root_config.root, atom, parent=parent) if dep_pkg is None: yield (atom, None) continue @@ -1819,9 +2643,14 @@ class depgraph(object): # Yield ~, =*, < and <= atoms first, since those are more likely to # cause slot conflicts, and we want those atoms to be displayed # in the resulting slot conflict message (see bug #291142). + # Give similar treatment to slot/sub-slot atoms. conflict_atoms = [] normal_atoms = [] + abi_atoms = [] for atom in cp_atoms: + if atom.slot_operator_built: + abi_atoms.append(atom) + continue conflict = False for child_pkg in atom_pkg_graph.child_nodes(atom): existing_node, matches = \ @@ -1834,7 +2663,7 @@ class depgraph(object): else: normal_atoms.append(atom) - for atom in chain(conflict_atoms, normal_atoms): + for atom in chain(abi_atoms, conflict_atoms, normal_atoms): child_pkgs = atom_pkg_graph.child_nodes(atom) # if more than one child, yield highest version if len(child_pkgs) > 1: @@ -1844,37 +2673,25 @@ class depgraph(object): def _queue_disjunctive_deps(self, pkg, dep_root, dep_priority, dep_struct): """ Queue disjunctive (virtual and ||) deps in self._dynamic_config._dep_disjunctive_stack. - Yields non-disjunctive deps. Raises InvalidDependString when + Yields non-disjunctive deps. Raises InvalidDependString when necessary. """ - i = 0 - while i < len(dep_struct): - x = dep_struct[i] + for x in dep_struct: if isinstance(x, list): - for y in self._queue_disjunctive_deps( - pkg, dep_root, dep_priority, x): - yield y - elif x == "||": - self._queue_disjunction(pkg, dep_root, dep_priority, - [ x, dep_struct[ i + 1 ] ] ) - i += 1 + if x and x[0] == "||": + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) + else: + for y in self._queue_disjunctive_deps( + pkg, dep_root, dep_priority, x): + yield y else: - try: - x = portage.dep.Atom(x, eapi=pkg.metadata["EAPI"]) - except portage.exception.InvalidAtom: - if not pkg.installed: - raise portage.exception.InvalidDependString( - "invalid atom: '%s'" % x) + # Note: Eventually this will check for PROPERTIES=virtual + # or whatever other metadata gets implemented for this + # purpose. + if x.cp.startswith('virtual/'): + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) else: - # Note: Eventually this will check for PROPERTIES=virtual - # or whatever other metadata gets implemented for this - # purpose. - if x.cp.startswith('virtual/'): - self._queue_disjunction( pkg, dep_root, - dep_priority, [ str(x) ] ) - else: - yield str(x) - i += 1 + yield x def _queue_disjunction(self, pkg, dep_root, dep_priority, dep_struct): self._dynamic_config._dep_disjunctive_stack.append( @@ -1887,10 +2704,8 @@ class depgraph(object): """ pkg, dep_root, dep_priority, dep_struct = \ self._dynamic_config._dep_disjunctive_stack.pop() - dep_string = portage.dep.paren_enclose(dep_struct, - unevaluated_atom=True) if not self._add_pkg_dep_string( - pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): + pkg, dep_root, dep_priority, dep_struct, allow_unsatisfied): return 0 return 1 @@ -1965,9 +2780,24 @@ class depgraph(object): continue yield arg, atom - def select_files(self, myfiles): + def select_files(self, args): + # Use the global event loop for spinner progress + # indication during file owner lookups (bug #461412). + spinner_id = None + try: + spinner = self._frozen_config.spinner + if spinner is not None and \ + spinner.update is not spinner.update_quiet: + spinner_id = self._event_loop.idle_add( + self._frozen_config.spinner.update) + return self._select_files(args) + finally: + if spinner_id is not None: + self._event_loop.source_remove(spinner_id) + + def _select_files(self, myfiles): """Given a list of .tbz2s, .ebuilds sets, and deps, populate - self._dynamic_config._initial_arg_list and call self._resolve to create the + self._dynamic_config._initial_arg_list and call self._resolve to create the appropriate depgraph and return a favorite list.""" self._load_vdb() debug = "--debug" in self._frozen_config.myopts @@ -2000,8 +2830,18 @@ class depgraph(object): writemsg("!!! Please ensure the tbz2 exists as specified.\n\n", noiselevel=-1) return 0, myfavorites mytbz2=portage.xpak.tbz2(x) - mykey=mytbz2.getelements("CATEGORY")[0]+"/"+os.path.splitext(os.path.basename(x))[0] - if os.path.realpath(x) != \ + mykey = None + cat = mytbz2.getfile("CATEGORY") + if cat is not None: + cat = _unicode_decode(cat.strip(), + encoding=_encodings['repo.content']) + mykey = cat + "/" + os.path.basename(x)[:-5] + + if mykey is None: + writemsg(colorize("BAD", "\n*** Package is missing CATEGORY metadata: %s.\n\n" % x), noiselevel=-1) + self._dynamic_config._skip_restart = True + return 0, myfavorites + elif os.path.realpath(x) != \ os.path.realpath(bindb.bintree.getname(mykey)): writemsg(colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n\n"), noiselevel=-1) self._dynamic_config._skip_restart = True @@ -2016,15 +2856,16 @@ class depgraph(object): pkgdir = os.path.dirname(ebuild_path) tree_root = os.path.dirname(os.path.dirname(pkgdir)) cp = pkgdir[len(tree_root)+1:] - e = portage.exception.PackageNotFound( - ("%s is not in a valid portage tree " + \ - "hierarchy or does not exist") % x) + error_msg = ("\n\n!!! '%s' is not in a valid portage tree " + "hierarchy or does not exist\n") % x if not portage.isvalidatom(cp): - raise e + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites cat = portage.catsplit(cp)[0] mykey = cat + "/" + os.path.basename(ebuild_path[:-7]) if not portage.isvalidatom("="+mykey): - raise e + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites ebuild_path = portdb.findname(mykey) if ebuild_path: if ebuild_path != os.path.join(os.path.realpath(tree_root), @@ -2040,8 +2881,8 @@ class depgraph(object): countdown(int(self._frozen_config.settings["EMERGE_WARNING_DELAY"]), "Continuing...") else: - raise portage.exception.PackageNotFound( - "%s is not in a valid portage tree hierarchy or does not exist" % x) + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites pkg = self._pkg(mykey, "ebuild", root_config, onlydeps=onlydeps, myrepo=portdb.getRepositoryName( os.path.dirname(os.path.dirname(os.path.dirname(ebuild_path))))) @@ -2074,6 +2915,30 @@ class depgraph(object): raise portage.exception.PackageSetNotFound(s) if s in depgraph_sets.sets: continue + + try: + set_atoms = root_config.setconfig.getSetAtoms(s) + except portage.exception.PackageSetNotFound as e: + writemsg_level("\n\n", level=logging.ERROR, + noiselevel=-1) + for pset in list(depgraph_sets.sets.values()) + [sets[s]]: + for error_msg in pset.errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + + writemsg_level(("emerge: the given set '%s' " + "contains a non-existent set named '%s'.\n") % \ + (s, e), level=logging.ERROR, noiselevel=-1) + if s in ('world', 'selected') and \ + SETPREFIX + e.value in sets['selected']: + writemsg_level(("Use `emerge --deselect %s%s` to " + "remove this set from world_sets.\n") % + (SETPREFIX, e,), level=logging.ERROR, + noiselevel=-1) + writemsg_level("\n", level=logging.ERROR, + noiselevel=-1) + return False, myfavorites + pset = sets[s] depgraph_sets.sets[s] = pset args.append(SetArg(arg=x, pset=pset, @@ -2093,7 +2958,7 @@ class depgraph(object): # came from, if any. # 2) It takes away freedom from the resolver to choose other # possible expansions when necessary. - if "/" in x: + if "/" in x.split(":")[0]: args.append(AtomArg(arg=x, atom=Atom(x, allow_repo=True), root_config=root_config)) continue @@ -2194,13 +3059,8 @@ class depgraph(object): return 0, [] for cpv in owners: - slot = vardb.aux_get(cpv, ["SLOT"])[0] - if not slot: - # portage now masks packages with missing slot, but it's - # possible that one was installed by an older version - atom = Atom(portage.cpv_getkey(cpv)) - else: - atom = Atom("%s:%s" % (portage.cpv_getkey(cpv), slot)) + pkg = vardb._pkg_str(cpv, None) + atom = Atom("%s:%s" % (pkg.cp, pkg.slot)) args.append(AtomArg(arg=atom, atom=atom, root_config=root_config)) @@ -2248,6 +3108,7 @@ class depgraph(object): args = revised_greedy_args del revised_greedy_args + args.extend(self._gen_reinstall_sets()) self._set_args(args) myfavorites = set(myfavorites) @@ -2255,7 +3116,8 @@ class depgraph(object): if isinstance(arg, (AtomArg, PackageArg)): myfavorites.add(arg.atom) elif isinstance(arg, SetArg): - myfavorites.add(arg.arg) + if not arg.internal: + myfavorites.add(arg.arg) myfavorites = list(myfavorites) if debug: @@ -2263,12 +3125,38 @@ class depgraph(object): # Order needs to be preserved since a feature of --nodeps # is to allow the user to force a specific merge order. self._dynamic_config._initial_arg_list = args[:] - + return self._resolve(myfavorites) - + + def _gen_reinstall_sets(self): + + atom_list = [] + for root, atom in self._rebuild.rebuild_list: + atom_list.append((root, '__auto_rebuild__', atom)) + for root, atom in self._rebuild.reinstall_list: + atom_list.append((root, '__auto_reinstall__', atom)) + for root, atom in self._dynamic_config._slot_operator_replace_installed: + atom_list.append((root, '__auto_slot_operator_replace_installed__', atom)) + + set_dict = {} + for root, set_name, atom in atom_list: + set_dict.setdefault((root, set_name), []).append(atom) + + for (root, set_name), atoms in set_dict.items(): + yield SetArg(arg=(SETPREFIX + set_name), + # Set reset_depth=False here, since we don't want these + # special sets to interact with depth calculations (see + # the emerge --deep=DEPTH option), though we want them + # to behave like normal arguments in most other respects. + pset=InternalPackageSet(initial_atoms=atoms), + force_reinstall=True, + internal=True, + reset_depth=False, + root_config=self._frozen_config.roots[root]) + def _resolve(self, myfavorites): - """Given self._dynamic_config._initial_arg_list, pull in the root nodes, - call self._creategraph to process theier deps and return + """Given self._dynamic_config._initial_arg_list, pull in the root nodes, + call self._creategraph to process theier deps and return a favorite list.""" debug = "--debug" in self._frozen_config.myopts onlydeps = "--onlydeps" in self._frozen_config.myopts @@ -2277,10 +3165,7 @@ class depgraph(object): pprovideddict = pkgsettings.pprovideddict virtuals = pkgsettings.getvirtuals() args = self._dynamic_config._initial_arg_list[:] - for root, atom in chain(self._rebuild.rebuild_list, - self._rebuild.reinstall_list): - args.append(AtomArg(arg=atom, atom=atom, - root_config=self._frozen_config.roots[root])) + for arg in self._expand_set_args(args, add_to_digraph=True): for atom in arg.pset.getAtoms(): self._spinner_update() @@ -2322,6 +3207,16 @@ class depgraph(object): if pprovided_match: continue + excluded = False + for any_match in self._iter_match_pkgs_any( + self._frozen_config.roots[myroot], atom): + if self._frozen_config.excluded_pkgs.findAtomForPackage( + any_match, modified_use=self._pkg_use_enabled(any_match)): + excluded = True + break + if excluded: + continue + if not (isinstance(arg, SetArg) and \ arg.name in ("selected", "system", "world")): self._dynamic_config._unsatisfied_deps_for_display.append( @@ -2390,22 +3285,12 @@ class depgraph(object): except self._unknown_internal_error: return False, myfavorites - digraph_set = frozenset(self._dynamic_config.digraph) - - if digraph_set.intersection( - self._dynamic_config._needed_unstable_keywords) or \ - digraph_set.intersection( - self._dynamic_config._needed_p_mask_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_use_config_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_license_changes) : - #We failed if the user needs to change the configuration - self._dynamic_config._success_without_autounmask = True + if (self._dynamic_config._slot_collision_info and + not self._accept_blocker_conflicts()) or \ + (self._dynamic_config._allow_backtracking and + "slot conflict" in self._dynamic_config._backtrack_infos): return False, myfavorites - digraph_set = None - if self._rebuild.trigger_rebuilds(): backtrack_infos = self._dynamic_config._backtrack_infos config = backtrack_infos.setdefault("config", {}) @@ -2414,6 +3299,68 @@ class depgraph(object): self._dynamic_config._need_restart = True return False, myfavorites + if "config" in self._dynamic_config._backtrack_infos and \ + ("slot_operator_mask_built" in self._dynamic_config._backtrack_infos["config"] or + "slot_operator_replace_installed" in self._dynamic_config._backtrack_infos["config"]) and \ + self.need_restart(): + return False, myfavorites + + if not self._dynamic_config._prune_rebuilds and \ + self._dynamic_config._slot_operator_replace_installed and \ + self._get_missed_updates(): + # When there are missed updates, we might have triggered + # some unnecessary rebuilds (see bug #439688). So, prune + # all the rebuilds and backtrack with the problematic + # updates masked. The next backtrack run should pull in + # any rebuilds that are really needed, and this + # prune_rebuilds path should never be entered more than + # once in a series of backtracking nodes (in order to + # avoid a backtracking loop). + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + config["prune_rebuilds"] = True + self._dynamic_config._need_restart = True + return False, myfavorites + + if self.need_restart(): + # want_restart_for_use_change triggers this + return False, myfavorites + + if "--fetchonly" not in self._frozen_config.myopts and \ + "--buildpkgonly" in self._frozen_config.myopts: + graph_copy = self._dynamic_config.digraph.copy() + removed_nodes = set() + for node in graph_copy: + if not isinstance(node, Package) or \ + node.operation == "nomerge": + removed_nodes.add(node) + graph_copy.difference_update(removed_nodes) + if not graph_copy.hasallzeros(ignore_priority = \ + DepPrioritySatisfiedRange.ignore_medium): + self._dynamic_config._buildpkgonly_deps_unsatisfied = True + self._dynamic_config._skip_restart = True + return False, myfavorites + + # Any failures except those due to autounmask *alone* should return + # before this point, since the success_without_autounmask flag that's + # set below is reserved for cases where there are *zero* other + # problems. For reference, see backtrack_depgraph, where it skips the + # get_best_run() call when success_without_autounmask is True. + + digraph_nodes = self._dynamic_config.digraph.nodes + + if any(x in digraph_nodes for x in + self._dynamic_config._needed_unstable_keywords) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_p_mask_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_use_config_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_license_changes) : + #We failed if the user needs to change the configuration + self._dynamic_config._success_without_autounmask = True + return False, myfavorites + # We're true here unless we are missing binaries. return (True, myfavorites) @@ -2455,8 +3402,8 @@ class depgraph(object): if refs is None: refs = [] atom_arg_map[atom_key] = refs - if arg not in refs: - refs.append(arg) + if arg not in refs: + refs.append(arg) for root in self._dynamic_config.sets: depgraph_sets = self._dynamic_config.sets[root] @@ -2486,14 +3433,15 @@ class depgraph(object): slots = set() for cpv in vardb.match(atom): # don't mix new virtuals with old virtuals - if portage.cpv_getkey(cpv) == highest_pkg.cp: - slots.add(vardb.aux_get(cpv, ["SLOT"])[0]) + pkg = vardb._pkg_str(cpv, None) + if pkg.cp == highest_pkg.cp: + slots.add(pkg.slot) - slots.add(highest_pkg.metadata["SLOT"]) + slots.add(highest_pkg.slot) if len(slots) == 1: return [] greedy_pkgs = [] - slots.remove(highest_pkg.metadata["SLOT"]) + slots.remove(highest_pkg.slot) while slots: slot = slots.pop() slot_atom = portage.dep.Atom("%s:%s" % (highest_pkg.cp, slot)) @@ -2507,9 +3455,9 @@ class depgraph(object): return [pkg.slot_atom for pkg in greedy_pkgs] blockers = {} - blocker_dep_keys = ["DEPEND", "PDEPEND", "RDEPEND"] + blocker_dep_keys = Package._dep_keys for pkg in greedy_pkgs + [highest_pkg]: - dep_str = " ".join(pkg.metadata[k] for k in blocker_dep_keys) + dep_str = " ".join(pkg._metadata[k] for k in blocker_dep_keys) try: selected_atoms = self._select_atoms( pkg.root, dep_str, self._pkg_use_enabled(pkg), @@ -2561,13 +3509,30 @@ class depgraph(object): not been scheduled for replacement. """ kwargs["trees"] = self._dynamic_config._graph_trees - return self._select_atoms_highest_available(*pargs, **kwargs) + return self._select_atoms_highest_available(*pargs, + **portage._native_kwargs(kwargs)) def _select_atoms_highest_available(self, root, depstring, myuse=None, parent=None, strict=True, trees=None, priority=None): """This will raise InvalidDependString if necessary. If trees is None then self._dynamic_config._filtered_trees is used.""" + if not isinstance(depstring, list): + eapi = None + is_valid_flag = None + if parent is not None: + eapi = parent.eapi + if not parent.installed: + is_valid_flag = parent.iuse.is_valid_flag + depstring = portage.dep.use_reduce(depstring, + uselist=myuse, opconvert=True, token_class=Atom, + is_valid_flag=is_valid_flag, eapi=eapi) + + if (self._dynamic_config.myparams.get( + "ignore_built_slot_operator_deps", "n") == "y" and + parent and parent.built): + ignore_built_slot_operator_deps(depstring) + pkgsettings = self._frozen_config.pkgsettings[root] if trees is None: trees = self._dynamic_config._filtered_trees @@ -2682,7 +3647,7 @@ class depgraph(object): return try: rdepend = self._select_atoms_from_graph( - pkg.root, pkg.metadata.get("RDEPEND", ""), + pkg.root, pkg._metadata.get("RDEPEND", ""), myuse=self._pkg_use_enabled(pkg), parent=pkg, strict=False) except InvalidDependString as e: @@ -2710,7 +3675,7 @@ class depgraph(object): """ try: rdepend = self._select_atoms( - pkg.root, pkg.metadata.get("RDEPEND", ""), + pkg.root, pkg._metadata.get("RDEPEND", ""), myuse=self._pkg_use_enabled(pkg), parent=pkg, priority=self._priority(runtime=True)) except InvalidDependString as e: @@ -2749,19 +3714,29 @@ class depgraph(object): child = None all_parents = self._dynamic_config._parent_atoms graph = self._dynamic_config.digraph + verbose_main_repo_display = "--verbose-main-repo-display" in \ + self._frozen_config.myopts + + def format_pkg(pkg): + pkg_name = "%s" % (pkg.cpv,) + if verbose_main_repo_display or pkg.repo != \ + pkg.root_config.settings.repositories.mainRepo().name: + pkg_name += _repo_separator + pkg.repo + return pkg_name if target_atom is not None and isinstance(node, Package): affecting_use = set() - for dep_str in "DEPEND", "RDEPEND", "PDEPEND": + for dep_str in Package._dep_keys: try: affecting_use.update(extract_affecting_use( - node.metadata[dep_str], target_atom, - eapi=node.metadata["EAPI"])) + node._metadata[dep_str], target_atom, + eapi=node.eapi)) except InvalidDependString: if not node.installed: raise affecting_use.difference_update(node.use.mask, node.use.force) - pkg_name = _unicode_decode("%s") % (node.cpv,) + pkg_name = format_pkg(node) + if affecting_use: usedep = [] for flag in affecting_use: @@ -2816,7 +3791,7 @@ class depgraph(object): node_type = "set" else: node_type = "argument" - dep_chain.append((_unicode_decode("%s") % (node,), node_type)) + dep_chain.append(("%s" % (node,), node_type)) elif node is not start_node: for ppkg, patom in all_parents[child]: @@ -2833,23 +3808,23 @@ class depgraph(object): if priorities is None: # This edge comes from _parent_atoms and was not added to # the graph, and _parent_atoms does not contain priorities. - dep_strings.add(node.metadata["DEPEND"]) - dep_strings.add(node.metadata["RDEPEND"]) - dep_strings.add(node.metadata["PDEPEND"]) + for k in Package._dep_keys: + dep_strings.add(node._metadata[k]) else: for priority in priorities: if priority.buildtime: - dep_strings.add(node.metadata["DEPEND"]) + for k in Package._buildtime_keys: + dep_strings.add(node._metadata[k]) if priority.runtime: - dep_strings.add(node.metadata["RDEPEND"]) + dep_strings.add(node._metadata["RDEPEND"]) if priority.runtime_post: - dep_strings.add(node.metadata["PDEPEND"]) + dep_strings.add(node._metadata["PDEPEND"]) affecting_use = set() for dep_str in dep_strings: try: affecting_use.update(extract_affecting_use( - dep_str, atom, eapi=node.metadata["EAPI"])) + dep_str, atom, eapi=node.eapi)) except InvalidDependString: if not node.installed: raise @@ -2858,7 +3833,7 @@ class depgraph(object): affecting_use.difference_update(node.use.mask, \ node.use.force) - pkg_name = _unicode_decode("%s") % (node.cpv,) + pkg_name = format_pkg(node) if affecting_use: usedep = [] for flag in affecting_use: @@ -2910,8 +3885,7 @@ class depgraph(object): if self._dynamic_config.digraph.parent_nodes(parent_arg): selected_parent = parent_arg else: - dep_chain.append( - (_unicode_decode("%s") % (parent_arg,), "argument")) + dep_chain.append(("%s" % (parent_arg,), "argument")) selected_parent = None node = selected_parent @@ -2926,7 +3900,7 @@ class depgraph(object): else: display_list.append("required by %s" % node) - msg = "#" + ", ".join(display_list) + "\n" + msg = "# " + "\n# ".join(display_list) + "\n" return msg @@ -2947,7 +3921,7 @@ class depgraph(object): if arg: xinfo='"%s"' % arg if isinstance(myparent, AtomArg): - xinfo = _unicode_decode('"%s"') % (myparent,) + xinfo = '"%s"' % (myparent,) # Discard null/ from failed cpv_expand category expansion. xinfo = xinfo.replace("null/", "") if root != self._frozen_config._running_root.root: @@ -2992,9 +3966,9 @@ class depgraph(object): repo = metadata.get('repository') pkg = self._pkg(cpv, pkg_type, root_config, installed=installed, myrepo=repo) - # pkg.metadata contains calculated USE for ebuilds, + # pkg._metadata contains calculated USE for ebuilds, # required later for getMissingLicenses. - metadata = pkg.metadata + metadata = pkg._metadata if pkg.invalid: # Avoid doing any operations with packages that # have invalid metadata. It would be unsafe at @@ -3033,12 +4007,13 @@ class depgraph(object): raise if not mreasons and \ not pkg.built and \ - pkg.metadata.get("REQUIRED_USE") and \ - eapi_has_required_use(pkg.metadata["EAPI"]): + pkg._metadata.get("REQUIRED_USE") and \ + eapi_has_required_use(pkg.eapi): if not check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag): + pkg.iuse.is_valid_flag, + eapi=pkg.eapi): required_use_unsatisfied.append(pkg) continue root_slot = (pkg.root, pkg.slot_atom) @@ -3083,12 +4058,12 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(pkg.use.mask, pkg.use.force)) - if untouchable_flags.intersection( + if any(x in untouchable_flags for x in chain(need_enable, need_disable)): continue missing_use_adjustable.add(pkg) - required_use = pkg.metadata.get("REQUIRED_USE") + required_use = pkg._metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(pkg) @@ -3097,8 +4072,10 @@ class depgraph(object): new_use.add(flag) for flag in need_disable: new_use.discard(flag) - if check_required_use(required_use, old_use, pkg.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): + if check_required_use(required_use, old_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi) \ + and not check_required_use(required_use, new_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi): required_use_warning = ", this change violates use flag constraints " + \ "defined by %s: '%s'" % (pkg.cpv, human_readable_required_use(required_use)) @@ -3133,10 +4110,10 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(myparent.use.mask, myparent.use.force)) - if untouchable_flags.intersection(involved_flags): + if any(x in untouchable_flags for x in involved_flags): continue - required_use = myparent.metadata.get("REQUIRED_USE") + required_use = myparent._metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(myparent) @@ -3146,8 +4123,12 @@ class depgraph(object): new_use.discard(flag) else: new_use.add(flag) - if check_required_use(required_use, old_use, myparent.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, myparent.iuse.is_valid_flag): + if check_required_use(required_use, old_use, + myparent.iuse.is_valid_flag, + eapi=myparent.eapi) and \ + not check_required_use(required_use, new_use, + myparent.iuse.is_valid_flag, + eapi=myparent.eapi): required_use_warning = ", this change violates use flag constraints " + \ "defined by %s: '%s'" % (myparent.cpv, \ human_readable_required_use(required_use)) @@ -3234,14 +4215,15 @@ class depgraph(object): writemsg("\n The following REQUIRED_USE flag constraints " + \ "are unsatisfied:\n", noiselevel=-1) reduced_noise = check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag).tounicode() + pkg.iuse.is_valid_flag, + eapi=pkg.eapi).tounicode() writemsg(" %s\n" % \ human_readable_required_use(reduced_noise), noiselevel=-1) normalized_required_use = \ - " ".join(pkg.metadata["REQUIRED_USE"].split()) + " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg("\n The above constraints " + \ "are a subset of the following complete expression:\n", @@ -3286,57 +4268,17 @@ class depgraph(object): not cp_exists and \ self._frozen_config.myopts.get( "--misspell-suggestions", "y") != "n": - cp = myparent.atom.cp.lower() - cat, pkg = portage.catsplit(cp) - if cat == "null": - cat = None writemsg("\nemerge: searching for similar names..." , noiselevel=-1) - all_cp = set() - all_cp.update(vardb.cp_all()) + dbs = [vardb] if "--usepkgonly" not in self._frozen_config.myopts: - all_cp.update(portdb.cp_all()) + dbs.append(portdb) if "--usepkg" in self._frozen_config.myopts: - all_cp.update(bindb.cp_all()) - # discard dir containing no ebuilds - all_cp.discard(cp) - - orig_cp_map = {} - for cp_orig in all_cp: - orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig) - all_cp = set(orig_cp_map) - - if cat: - matches = difflib.get_close_matches(cp, all_cp) - else: - pkg_to_cp = {} - for other_cp in list(all_cp): - other_pkg = portage.catsplit(other_cp)[1] - if other_pkg == pkg: - # Check for non-identical package that - # differs only by upper/lower case. - identical = True - for cp_orig in orig_cp_map[other_cp]: - if portage.catsplit(cp_orig)[1] != \ - portage.catsplit(atom.cp)[1]: - identical = False - break - if identical: - # discard dir containing no ebuilds - all_cp.discard(other_cp) - continue - pkg_to_cp.setdefault(other_pkg, set()).add(other_cp) - pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp) - matches = [] - for pkg_match in pkg_matches: - matches.extend(pkg_to_cp[pkg_match]) + dbs.append(bindb) - matches_orig_case = [] - for cp in matches: - matches_orig_case.extend(orig_cp_map[cp]) - matches = matches_orig_case + matches = similar_name_search(dbs, atom) if len(matches) == 1: writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" @@ -3357,8 +4299,7 @@ class depgraph(object): dep_chain = self._get_dep_chain(myparent, atom) for node, node_type in dep_chain: msg.append('(dependency required by "%s" [%s])' % \ - (colorize('INFORM', _unicode_decode("%s") % \ - (node)), node_type)) + (colorize('INFORM', "%s" % (node)), node_type)) if msg: writemsg("\n".join(msg), noiselevel=-1) @@ -3436,7 +4377,8 @@ class depgraph(object): # the newly built package still won't have the expected slot. # Therefore, assume that such SLOT dependencies are already # satisfied rather than forcing a rebuild. - if not matched_something and installed and atom.slot is not None: + if not matched_something and installed and \ + atom.slot is not None and not atom.slot_operator_built: if "remove" in self._dynamic_config.myparams: # We need to search the portdbapi, which is not in our @@ -3460,11 +4402,11 @@ class depgraph(object): for other_db, other_type, other_built, \ other_installed, other_keys in dbs: try: - if atom.slot == \ - other_db.aux_get(cpv, ["SLOT"])[0]: + if portage.dep._match_slot(atom, + other_db._pkg_str(_unicode(cpv), None)): slot_available = True break - except KeyError: + except (KeyError, InvalidData): pass if not slot_available: continue @@ -3476,20 +4418,12 @@ class depgraph(object): yield inst_pkg return - def _select_pkg_highest_available(self, root, atom, onlydeps=False): - cache_key = (root, atom, atom.unevaluated_atom, onlydeps) + def _select_pkg_highest_available(self, root, atom, onlydeps=False, parent=None): + cache_key = (root, atom, atom.unevaluated_atom, onlydeps, self._dynamic_config._autounmask) ret = self._dynamic_config._highest_pkg_cache.get(cache_key) if ret is not None: - pkg, existing = ret - if pkg and not existing: - existing = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) - if existing and existing == pkg: - # Update the cache to reflect that the - # package has been added to the graph. - ret = pkg, pkg - self._dynamic_config._highest_pkg_cache[cache_key] = ret return ret - ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) + ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps, parent=parent) self._dynamic_config._highest_pkg_cache[cache_key] = ret pkg, existing = ret if pkg is not None: @@ -3504,21 +4438,85 @@ class depgraph(object): True if the user has not explicitly requested for this package to be replaced (typically via an atom on the command line). """ - if "selective" not in self._dynamic_config.myparams and \ - pkg.root == self._frozen_config.target_root: - if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, - modified_use=self._pkg_use_enabled(pkg)): - return True - try: - next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - pass - else: + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + return True + + arg = False + try: + for arg, atom in self._iter_atoms_for_pkg(pkg): + if arg.force_reinstall: + return False + except InvalidDependString: + pass + + if "selective" in self._dynamic_config.myparams: + return True + + return not arg + + def _want_update_pkg(self, parent, pkg): + + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + return False + + arg_atoms = None + try: + arg_atoms = list(self._iter_atoms_for_pkg(pkg)) + except InvalidDependString: + if not pkg.installed: + # should have been masked before it was selected + raise + + depth = parent.depth or 0 + depth += 1 + + if arg_atoms: + for arg, atom in arg_atoms: + if arg.reset_depth: + depth = 0 + break + + deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + return (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + def _equiv_ebuild_visible(self, pkg, autounmask_level=None): + try: + pkg_eb = self._pkg( + pkg.cpv, "ebuild", pkg.root_config, myrepo=pkg.repo) + except portage.exception.PackageNotFound: + pkg_eb_visible = False + for pkg_eb in self._iter_match_pkgs(pkg.root_config, + "ebuild", Atom("=%s" % (pkg.cpv,))): + if self._pkg_visibility_check(pkg_eb, autounmask_level): + pkg_eb_visible = True + break + if not pkg_eb_visible: return False + else: + if not self._pkg_visibility_check(pkg_eb, autounmask_level): + return False + return True + def _equiv_binary_installed(self, pkg): + build_time = pkg.build_time + if not build_time: + return False + + try: + inst_pkg = self._pkg(pkg.cpv, "installed", + pkg.root_config, installed=True) + except PackageNotFound: + return False + + return build_time == inst_pkg.build_time + class _AutounmaskLevel(object): __slots__ = ("allow_use_changes", "allow_unstable_keywords", "allow_license_changes", \ "allow_missing_keywords", "allow_unmasks") @@ -3534,11 +4532,13 @@ class depgraph(object): """ Iterate over the different allowed things to unmask. - 1. USE + 0. USE + 1. USE + license 2. USE + ~arch + license 3. USE + ~arch + license + missing keywords - 4. USE + ~arch + license + masks - 5. USE + ~arch + license + missing keywords + masks + 4. USE + license + masks + 5. USE + ~arch + license + masks + 6. USE + ~arch + license + missing keywords + masks Some thoughts: * Do least invasive changes first. @@ -3553,16 +4553,30 @@ class depgraph(object): autounmask_level = self._AutounmaskLevel() autounmask_level.allow_use_changes = True + yield autounmask_level - for only_use_changes in (True, False): + autounmask_level.allow_license_changes = True + yield autounmask_level - autounmask_level.allow_unstable_keywords = (not only_use_changes) - autounmask_level.allow_license_changes = (not only_use_changes) + autounmask_level.allow_unstable_keywords = True + yield autounmask_level - for missing_keyword, unmask in ((False,False), (True, False), (False, True), (True, True)): + if not autounmask_keep_masks: - if (only_use_changes or autounmask_keep_masks) and (missing_keyword or unmask): - break + autounmask_level.allow_missing_keywords = True + yield autounmask_level + + # 4. USE + license + masks + # Try to respect keywords while discarding + # package.mask (see bug #463394). + autounmask_level.allow_unstable_keywords = False + autounmask_level.allow_missing_keywords = False + autounmask_level.allow_unmasks = True + yield autounmask_level + + autounmask_level.allow_unstable_keywords = True + + for missing_keyword, unmask in ((False, True), (True, True)): autounmask_level.allow_missing_keywords = missing_keyword autounmask_level.allow_unmasks = unmask @@ -3570,33 +4584,42 @@ class depgraph(object): yield autounmask_level - def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False): - pkg, existing = self._wrapped_select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) + def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False, parent=None): + pkg, existing = self._wrapped_select_pkg_highest_available_imp( + root, atom, onlydeps=onlydeps, parent=parent) default_selection = (pkg, existing) - def reset_pkg(pkg): + if self._dynamic_config._autounmask is True: if pkg is not None and \ pkg.installed and \ not self._want_installed_pkg(pkg): pkg = None - if self._dynamic_config._autounmask is True: - reset_pkg(pkg) + # Temporarily reset _need_restart state, in order to + # avoid interference as reported in bug #459832. + earlier_need_restart = self._dynamic_config._need_restart + self._dynamic_config._need_restart = False + try: + for autounmask_level in self._autounmask_levels(): + if pkg is not None: + break - for autounmask_level in self._autounmask_levels(): - if pkg is not None: - break + pkg, existing = \ + self._wrapped_select_pkg_highest_available_imp( + root, atom, onlydeps=onlydeps, + autounmask_level=autounmask_level, parent=parent) - pkg, existing = \ - self._wrapped_select_pkg_highest_available_imp( - root, atom, onlydeps=onlydeps, - autounmask_level=autounmask_level) + if pkg is not None and \ + pkg.installed and \ + not self._want_installed_pkg(pkg): + pkg = None - reset_pkg(pkg) - - if self._dynamic_config._need_restart: - return None, None + if self._dynamic_config._need_restart: + return None, None + finally: + if earlier_need_restart: + self._dynamic_config._need_restart = True if pkg is None: # This ensures that we can fall back to an installed package @@ -3726,25 +4749,29 @@ class depgraph(object): new_changes = {} for flag, state in target_use.items(): + real_flag = pkg.iuse.get_real_flag(flag) + if real_flag is None: + # Triggered by use-dep defaults. + continue if state: - if flag not in old_use: - if new_changes.get(flag) == False: + if real_flag not in old_use: + if new_changes.get(real_flag) == False: return old_use - new_changes[flag] = True + new_changes[real_flag] = True new_use.add(flag) else: - if flag in old_use: - if new_changes.get(flag) == True: + if real_flag in old_use: + if new_changes.get(real_flag) == True: return old_use - new_changes[flag] = False + new_changes[real_flag] = False new_use.update(old_use.difference(target_use)) def want_restart_for_use_change(pkg, new_use): if pkg not in self._dynamic_config.digraph.nodes: return False - for key in "DEPEND", "RDEPEND", "PDEPEND", "LICENSE": - dep = pkg.metadata[key] + for key in Package._dep_keys + ("LICENSE",): + dep = pkg._metadata[key] old_val = set(portage.dep.use_reduce(dep, pkg.use.enabled, is_valid_flag=pkg.iuse.is_valid_flag, flat=True)) new_val = set(portage.dep.use_reduce(dep, new_use, is_valid_flag=pkg.iuse.is_valid_flag, flat=True)) @@ -3758,7 +4785,7 @@ class depgraph(object): new_use, changes = self._dynamic_config._needed_use_config_changes.get(pkg) for ppkg, atom in parent_atoms: if not atom.use or \ - not atom.use.required.intersection(changes): + not any(x in atom.use.required for x in changes): continue else: return True @@ -3767,13 +4794,15 @@ class depgraph(object): if new_changes != old_changes: #Don't do the change if it violates REQUIRED_USE. - required_use = pkg.metadata.get("REQUIRED_USE") - if required_use and check_required_use(required_use, old_use, pkg.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): + required_use = pkg._metadata.get("REQUIRED_USE") + if required_use and check_required_use(required_use, old_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi) and \ + not check_required_use(required_use, new_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi): return old_use - if pkg.use.mask.intersection(new_changes) or \ - pkg.use.force.intersection(new_changes): + if any(x in pkg.use.mask for x in new_changes) or \ + any(x in pkg.use.force for x in new_changes): return old_use self._dynamic_config._needed_use_config_changes[pkg] = (new_use, new_changes) @@ -3785,14 +4814,13 @@ class depgraph(object): self._dynamic_config._need_restart = True return new_use - def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, autounmask_level=None): + def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, autounmask_level=None, parent=None): root_config = self._frozen_config.roots[root] pkgsettings = self._frozen_config.pkgsettings[root] dbs = self._dynamic_config._filtered_trees[root]["dbs"] vardb = self._frozen_config.roots[root].trees["vartree"].dbapi # List of acceptable packages, ordered by type preference. matched_packages = [] - matched_pkgs_ignore_use = [] highest_version = None if not isinstance(atom, portage.dep.Atom): atom = portage.dep.Atom(atom) @@ -3844,7 +4872,7 @@ class depgraph(object): # Ignore USE deps for the initial match since we want to # ensure that updates aren't missed solely due to the user's # USE configuration. - for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, + for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, onlydeps=onlydeps): if pkg.cp != atom_cp and have_new_virt: # pull in a new-style virtual instead @@ -3930,8 +4958,8 @@ class depgraph(object): for selected_pkg in matched_packages: if selected_pkg.type_name == "binary" and \ selected_pkg.cpv == pkg.cpv and \ - selected_pkg.metadata.get('BUILD_TIME') == \ - pkg.metadata.get('BUILD_TIME'): + selected_pkg.build_time == \ + pkg.build_time: identical_binary = True break @@ -3944,37 +4972,24 @@ class depgraph(object): if not use_ebuild_visibility and (usepkgonly or useoldpkg): if pkg.installed and pkg.masks: continue - else: - try: - pkg_eb = self._pkg( - pkg.cpv, "ebuild", root_config, myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._iter_match_pkgs(pkg.root_config, - "ebuild", Atom("=%s" % (pkg.cpv,))): - if self._pkg_visibility_check(pkg_eb, autounmask_level): - pkg_eb_visible = True - break - if not pkg_eb_visible: - continue - else: - if not self._pkg_visibility_check(pkg_eb, autounmask_level): - continue + elif not self._equiv_ebuild_visible(pkg, + autounmask_level=autounmask_level): + continue # Calculation of USE for unbuilt ebuilds is relatively # expensive, so it is only performed lazily, after the # above visibility checks are complete. myarg = None - if root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - if not installed: - # masked by corruption - continue + try: + for myarg, myarg_atom in self._iter_atoms_for_pkg(pkg): + if myarg.force_reinstall: + reinstall = True + break + except InvalidDependString: + if not installed: + # masked by corruption + continue if not installed and myarg: found_available_arg = True @@ -3987,7 +5002,6 @@ class depgraph(object): if atom.use: - matched_pkgs_ignore_use.append(pkg) if autounmask_level and autounmask_level.allow_use_changes and not pkg.built: target_use = {} for flag in atom.use.enabled: @@ -4000,11 +5014,14 @@ class depgraph(object): use_match = True can_adjust_use = not pkg.built - missing_enabled = atom.use.missing_enabled.difference(pkg.iuse.all) - missing_disabled = atom.use.missing_disabled.difference(pkg.iuse.all) + is_valid_flag = pkg.iuse.is_valid_flag + missing_enabled = frozenset(x for x in + atom.use.missing_enabled if not is_valid_flag(x)) + missing_disabled = frozenset(x for x in + atom.use.missing_disabled if not is_valid_flag(x)) if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + if any(x in atom.use.enabled for x in missing_disabled): use_match = False can_adjust_use = False need_enabled = atom.use.enabled.difference(use) @@ -4013,11 +5030,11 @@ class depgraph(object): if need_enabled: use_match = False if can_adjust_use: - if pkg.use.mask.intersection(need_enabled): + if any(x in pkg.use.mask for x in need_enabled): can_adjust_use = False if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if any(x in atom.use.disabled for x in missing_enabled): use_match = False can_adjust_use = False need_disabled = atom.use.disabled.intersection(use) @@ -4026,8 +5043,8 @@ class depgraph(object): if need_disabled: use_match = False if can_adjust_use: - if pkg.use.force.difference( - pkg.use.mask).intersection(need_disabled): + if any(x in pkg.use.force and x not in + pkg.use.mask for x in need_disabled): can_adjust_use = False if not use_match: @@ -4075,7 +5092,11 @@ class depgraph(object): break # Compare built package to current config and # reject the built package if necessary. - if built and not useoldpkg and (not installed or matched_pkgs_ignore_use) and \ + if built and not useoldpkg and \ + (not installed or matched_packages) and \ + not (installed and + self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg))) and \ ("--newuse" in self._frozen_config.myopts or \ "--reinstall" in self._frozen_config.myopts or \ (not installed and self._dynamic_config.myparams.get( @@ -4160,6 +5181,26 @@ class depgraph(object): return existing_node, existing_node if len(matched_packages) > 1: + if parent is not None and \ + (parent.root, parent.slot_atom) in self._dynamic_config._slot_operator_replace_installed: + # We're forcing a rebuild of the parent because we missed some + # update because of a slot operator dep. + if atom.slot_operator == "=" and atom.sub_slot is None: + # This one is a slot operator dep. Exclude the installed packages if a newer non-installed + # pkg exists. + highest_installed = None + for pkg in matched_packages: + if pkg.installed: + if highest_installed is None or pkg.version > highest_installed.version: + highest_installed = pkg + + if highest_installed: + non_installed = [pkg for pkg in matched_packages \ + if not pkg.installed and pkg.version > highest_installed.version] + + if non_installed: + matched_packages = non_installed + if rebuilt_binaries: inst_pkg = None built_pkg = None @@ -4177,15 +5218,8 @@ class depgraph(object): # non-empty, in order to avoid cases like to # bug #306659 where BUILD_TIME fields are missing # in local and/or remote Packages file. - try: - built_timestamp = int(built_pkg.metadata['BUILD_TIME']) - except (KeyError, ValueError): - built_timestamp = 0 - - try: - installed_timestamp = int(inst_pkg.metadata['BUILD_TIME']) - except (KeyError, ValueError): - installed_timestamp = 0 + built_timestamp = built_pkg.build_time + installed_timestamp = inst_pkg.build_time if unbuilt_pkg is not None and unbuilt_pkg > built_pkg: pass @@ -4232,7 +5266,7 @@ class depgraph(object): # ordered by type preference ("ebuild" type is the last resort) return matched_packages[-1], existing_node - def _select_pkg_from_graph(self, root, atom, onlydeps=False): + def _select_pkg_from_graph(self, root, atom, onlydeps=False, parent=None): """ Select packages that have already been added to the graph or those that are installed and have not been scheduled for @@ -4242,11 +5276,18 @@ class depgraph(object): matches = graph_db.match_pkgs(atom) if not matches: return None, None - pkg = matches[-1] # highest match - in_graph = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) - return pkg, in_graph - def _select_pkg_from_installed(self, root, atom, onlydeps=False): + # There may be multiple matches, and they may + # conflict with eachother, so choose the highest + # version that has already been added to the graph. + for pkg in reversed(matches): + if pkg in self._dynamic_config.digraph: + return pkg, pkg + + # Fall back to installed packages + return self._select_pkg_from_installed(root, atom, onlydeps=onlydeps, parent=parent) + + def _select_pkg_from_installed(self, root, atom, onlydeps=False, parent=None): """ Select packages that are installed. """ @@ -4269,6 +5310,14 @@ class depgraph(object): unmasked = [pkg for pkg in matches if not pkg.masks] if unmasked: matches = unmasked + if len(matches) > 1: + # Now account for packages for which existing + # ebuilds are masked or unavailable (bug #445506). + unmasked = [pkg for pkg in matches if + self._equiv_ebuild_visible(pkg)] + if unmasked: + matches = unmasked + pkg = matches[-1] # highest match in_graph = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) return pkg, in_graph @@ -4293,9 +5342,19 @@ class depgraph(object): "recurse" not in self._dynamic_config.myparams: return 1 + complete_if_new_use = self._dynamic_config.myparams.get( + "complete_if_new_use", "y") == "y" + complete_if_new_ver = self._dynamic_config.myparams.get( + "complete_if_new_ver", "y") == "y" + rebuild_if_new_slot = self._dynamic_config.myparams.get( + "rebuild_if_new_slot", "y") == "y" + complete_if_new_slot = rebuild_if_new_slot + if "complete" not in self._dynamic_config.myparams and \ - self._dynamic_config.myparams.get("complete_if_new_ver", "y") == "y": - # Enable complete mode if an installed package version will change. + (complete_if_new_use or + complete_if_new_ver or complete_if_new_slot): + # Enable complete mode if an installed package will change somehow. + use_change = False version_change = False for node in self._dynamic_config.digraph: if not isinstance(node, Package) or \ @@ -4303,12 +5362,42 @@ class depgraph(object): continue vardb = self._frozen_config.roots[ node.root].trees["vartree"].dbapi - inst_pkg = vardb.match_pkgs(node.slot_atom) - if inst_pkg and (inst_pkg[0] > node or inst_pkg[0] < node): - version_change = True - break - if version_change: + if complete_if_new_use or complete_if_new_ver: + inst_pkg = vardb.match_pkgs(node.slot_atom) + if inst_pkg and inst_pkg[0].cp == node.cp: + inst_pkg = inst_pkg[0] + if complete_if_new_ver: + if inst_pkg < node or node < inst_pkg: + version_change = True + break + elif not (inst_pkg.slot == node.slot and + inst_pkg.sub_slot == node.sub_slot): + # slot/sub-slot change without revbump gets + # similar treatment to a version change + version_change = True + break + + # Intersect enabled USE with IUSE, in order to + # ignore forced USE from implicit IUSE flags, since + # they're probably irrelevant and they are sensitive + # to use.mask/force changes in the profile. + if complete_if_new_use and \ + (node.iuse.all != inst_pkg.iuse.all or + self._pkg_use_enabled(node).intersection(node.iuse.all) != + self._pkg_use_enabled(inst_pkg).intersection(inst_pkg.iuse.all)): + use_change = True + break + + if complete_if_new_slot: + cp_list = vardb.match_pkgs(Atom(node.cp)) + if (cp_list and cp_list[0].cp == node.cp and + not any(node.slot == pkg.slot and + node.sub_slot == pkg.sub_slot for pkg in cp_list)): + version_change = True + break + + if use_change or version_change: self._dynamic_config.myparams["complete"] = True if "complete" not in self._dynamic_config.myparams: @@ -4322,6 +5411,7 @@ class depgraph(object): # scheduled for replacement. Also, toggle the "deep" # parameter so that all dependencies are traversed and # accounted for. + self._dynamic_config._complete_mode = True self._select_atoms = self._select_atoms_from_graph if "remove" in self._dynamic_config.myparams: self._select_package = self._select_pkg_from_installed @@ -4409,7 +5499,7 @@ class depgraph(object): return 0 return 1 - def _pkg(self, cpv, type_name, root_config, installed=False, + def _pkg(self, cpv, type_name, root_config, installed=False, onlydeps=False, myrepo = None): """ Get a package instance from the cache, or create a new @@ -4480,7 +5570,7 @@ class depgraph(object): # For installed packages, always ignore blockers from DEPEND since # only runtime dependencies should be relevant for packages that # are already built. - dep_keys = ["RDEPEND", "PDEPEND"] + dep_keys = Package._runtime_keys for myroot in self._frozen_config.trees: if self._frozen_config.myopts.get("--root-deps") is not None and \ @@ -4542,7 +5632,7 @@ class depgraph(object): self._spinner_update() blocker_data = blocker_cache.get(cpv) if blocker_data is not None and \ - blocker_data.counter != long(pkg.metadata["COUNTER"]): + blocker_data.counter != pkg.counter: blocker_data = None # If blocker data from the graph is available, use @@ -4559,9 +5649,8 @@ class depgraph(object): blockers is not None: # Re-use the blockers from the graph. blocker_atoms = sorted(blockers) - counter = long(pkg.metadata["COUNTER"]) blocker_data = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(pkg.counter, blocker_atoms) blocker_cache[pkg.cpv] = blocker_data continue @@ -4586,7 +5675,7 @@ class depgraph(object): # matches (this can happen if an atom lacks a # category). show_invalid_depstring_notice( - pkg, depstr, _unicode_decode("%s") % (e,)) + pkg, depstr, "%s" % (e,)) del e raise if not success: @@ -4603,22 +5692,20 @@ class depgraph(object): blocker_atoms = [myatom for myatom in atoms \ if myatom.blocker] blocker_atoms.sort() - counter = long(pkg.metadata["COUNTER"]) blocker_cache[cpv] = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(pkg.counter, blocker_atoms) if blocker_atoms: try: for atom in blocker_atoms: blocker = Blocker(atom=atom, - eapi=pkg.metadata["EAPI"], + eapi=pkg.eapi, priority=self._priority(runtime=True), root=myroot) self._dynamic_config._blocker_parents.add(blocker, pkg) except portage.exception.InvalidAtom as e: depstr = " ".join(vardb.aux_get(pkg.cpv, dep_keys)) show_invalid_depstring_notice( - pkg, depstr, - _unicode_decode("Invalid Atom: %s") % (e,)) + pkg, depstr, "Invalid Atom: %s" % (e,)) return False for cpv in stale_cache: del blocker_cache[cpv] @@ -4640,7 +5727,7 @@ class depgraph(object): myroot = blocker.root initial_db = self._frozen_config.trees[myroot]["vartree"].dbapi final_db = self._dynamic_config.mydbapi[myroot] - + provider_virtual = False if blocker.cp in virtuals and \ not self._have_new_virt(blocker.root, blocker.cp): @@ -4751,7 +5838,7 @@ class depgraph(object): for inst_pkg, inst_task in depends_on_order: uninst_task = Package(built=inst_pkg.built, cpv=inst_pkg.cpv, installed=inst_pkg.installed, - metadata=inst_pkg.metadata, + metadata=inst_pkg._metadata, operation="uninstall", root_config=inst_pkg.root_config, type_name=inst_pkg.type_name) @@ -4817,7 +5904,12 @@ class depgraph(object): mygraph.order.sort(key=cmp_sort_key(cmp_merge_preference)) - def altlist(self, reversed=False): + def altlist(self, reversed=DeprecationWarning): + + if reversed is not DeprecationWarning: + warnings.warn("The reversed parameter of " + "_emerge.depgraph.depgraph.altlist() is deprecated", + DeprecationWarning, stacklevel=2) while self._dynamic_config._serialized_tasks_cache is None: self._resolve_conflicts() @@ -4827,9 +5919,13 @@ class depgraph(object): except self._serialize_tasks_retry: pass - retlist = self._dynamic_config._serialized_tasks_cache[:] - if reversed: + retlist = self._dynamic_config._serialized_tasks_cache + if reversed is not DeprecationWarning and reversed: + # TODO: remove the "reversed" parameter (builtin name collision) + retlist = list(retlist) retlist.reverse() + retlist = tuple(retlist) + return retlist def _implicit_libc_deps(self, mergelist, graph): @@ -4937,16 +6033,27 @@ class depgraph(object): root_config.root]["root_config"] = root_config def _resolve_conflicts(self): + + if "complete" not in self._dynamic_config.myparams and \ + self._dynamic_config._allow_backtracking and \ + self._dynamic_config._slot_collision_nodes and \ + not self._accept_blocker_conflicts(): + self._dynamic_config.myparams["complete"] = True + if not self._complete_graph(): raise self._unknown_internal_error() + self._process_slot_conflicts() + + if self._dynamic_config._allow_backtracking: + self._slot_operator_trigger_reinstalls() + if not self._validate_blockers(): - self._dynamic_config._skip_restart = True + # Blockers don't trigger the _skip_restart flag, since + # backtracking may solve blockers when it solves slot + # conflicts (or by blind luck). raise self._unknown_internal_error() - if self._dynamic_config._slot_collision_info: - self._process_slot_conflicts() - def _serialize_tasks(self): debug = "--debug" in self._frozen_config.myopts @@ -5061,7 +6168,7 @@ class depgraph(object): if running_portage is not None: try: portage_rdepend = self._select_atoms_highest_available( - running_root, running_portage.metadata["RDEPEND"], + running_root, running_portage._metadata["RDEPEND"], myuse=self._pkg_use_enabled(running_portage), parent=running_portage, strict=False) except portage.exception.InvalidDependString as e: @@ -5241,7 +6348,7 @@ class depgraph(object): for node in nodes: parents = mygraph.parent_nodes(node, ignore_priority=DepPrioritySatisfiedRange.ignore_soft) - if parents and set(parents).intersection(asap_nodes): + if any(x in asap_nodes for x in parents): selected_nodes = [node] break else: @@ -5409,8 +6516,7 @@ class depgraph(object): other_version = None for pkg in vardb.match_pkgs(atom): if pkg.cpv == task.cpv and \ - pkg.metadata["COUNTER"] == \ - task.metadata["COUNTER"]: + pkg.counter == task.counter: continue other_version = pkg break @@ -5617,7 +6723,7 @@ class depgraph(object): inst_pkg = inst_pkg[0] uninst_task = Package(built=inst_pkg.built, cpv=inst_pkg.cpv, installed=inst_pkg.installed, - metadata=inst_pkg.metadata, + metadata=inst_pkg._metadata, operation="uninstall", root_config=inst_pkg.root_config, type_name=inst_pkg.type_name) @@ -5689,17 +6795,21 @@ class depgraph(object): for blocker in unsolvable_blockers: retlist.append(blocker) + retlist = tuple(retlist) + if unsolvable_blockers and \ not self._accept_blocker_conflicts(): self._dynamic_config._unsatisfied_blockers_for_display = unsolvable_blockers - self._dynamic_config._serialized_tasks_cache = retlist[:] + self._dynamic_config._serialized_tasks_cache = retlist self._dynamic_config._scheduler_graph = scheduler_graph - self._dynamic_config._skip_restart = True + # Blockers don't trigger the _skip_restart flag, since + # backtracking may solve blockers when it solves slot + # conflicts (or by blind luck). raise self._unknown_internal_error() if self._dynamic_config._slot_collision_info and \ not self._accept_blocker_conflicts(): - self._dynamic_config._serialized_tasks_cache = retlist[:] + self._dynamic_config._serialized_tasks_cache = retlist self._dynamic_config._scheduler_graph = scheduler_graph raise self._unknown_internal_error() @@ -5753,13 +6863,8 @@ class depgraph(object): def _show_merge_list(self): if self._dynamic_config._serialized_tasks_cache is not None and \ not (self._dynamic_config._displayed_list is not None and \ - (self._dynamic_config._displayed_list == self._dynamic_config._serialized_tasks_cache or \ - self._dynamic_config._displayed_list == \ - list(reversed(self._dynamic_config._serialized_tasks_cache)))): - display_list = self._dynamic_config._serialized_tasks_cache[:] - if "--tree" in self._frozen_config.myopts: - display_list.reverse() - self.display(display_list) + self._dynamic_config._displayed_list is self._dynamic_config._serialized_tasks_cache): + self.display(self._dynamic_config._serialized_tasks_cache) def _show_unsatisfied_blockers(self, blockers): self._show_merge_list() @@ -5777,10 +6882,18 @@ class depgraph(object): # the reasons are not apparent from the normal merge list # display. + slot_collision_info = self._dynamic_config._slot_collision_info + conflict_pkgs = {} for blocker in blockers: for pkg in chain(self._dynamic_config._blocked_pkgs.child_nodes(blocker), \ self._dynamic_config._blocker_parents.parent_nodes(blocker)): + if (pkg.slot_atom, pkg.root) in slot_collision_info: + # The slot conflict display has better noise reduction + # than the unsatisfied blockers display, so skip + # unsatisfied blockers display for packages involved + # directly in slot conflicts (see bug #385391). + continue parent_atoms = self._dynamic_config._parent_atoms.get(pkg) if not parent_atoms: atom = self._dynamic_config._blocked_world_pkgs.get(pkg) @@ -5838,7 +6951,14 @@ class depgraph(object): else: # Display the specific atom from SetArg or # Package types. - msg.append("%s required by %s" % (atom, parent)) + if atom != atom.unevaluated_atom: + # Show the unevaluated atom, since it can reveal + # issues with conditional use-flags missing + # from IUSE. + msg.append("%s (%s) required by %s" % + (atom.unevaluated_atom, atom, parent)) + else: + msg.append("%s required by %s" % (atom, parent)) msg.append("\n") msg.append("\n") @@ -5854,6 +6974,10 @@ class depgraph(object): # redundantly displaying this exact same merge list # again via _show_merge_list(). self._dynamic_config._displayed_list = mylist + + if "--tree" in self._frozen_config.myopts: + mylist = tuple(reversed(mylist)) + display = Display() return display(self, mylist, favorites, verbosity) @@ -5926,7 +7050,7 @@ class depgraph(object): if is_latest: unstable_keyword_msg[root].append(">=%s %s\n" % (pkg.cpv, keyword)) elif is_latest_in_slot: - unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], keyword)) + unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, keyword)) else: unstable_keyword_msg[root].append("=%s %s\n" % (pkg.cpv, keyword)) else: @@ -5949,7 +7073,7 @@ class depgraph(object): keyword = reason.unmask_hint.value comment, filename = portage.getmaskingreason( - pkg.cpv, metadata=pkg.metadata, + pkg.cpv, metadata=pkg._metadata, settings=pkgsettings, portdb=pkg.root_config.trees["porttree"].dbapi, return_location=True) @@ -5966,7 +7090,7 @@ class depgraph(object): if is_latest: p_mask_change_msg[root].append(">=%s\n" % pkg.cpv) elif is_latest_in_slot: - p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.metadata["SLOT"])) + p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.slot)) else: p_mask_change_msg[root].append("=%s\n" % pkg.cpv) else: @@ -5991,7 +7115,7 @@ class depgraph(object): if is_latest: use_changes_msg[root].append(">=%s %s\n" % (pkg.cpv, " ".join(adjustments))) elif is_latest_in_slot: - use_changes_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], " ".join(adjustments))) + use_changes_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, " ".join(adjustments))) else: use_changes_msg[root].append("=%s %s\n" % (pkg.cpv, " ".join(adjustments))) @@ -6008,7 +7132,7 @@ class depgraph(object): if is_latest: license_msg[root].append(">=%s %s\n" % (pkg.cpv, " ".join(sorted(missing_licenses)))) elif is_latest_in_slot: - license_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], " ".join(sorted(missing_licenses)))) + license_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, " ".join(sorted(missing_licenses)))) else: license_msg[root].append("=%s %s\n" % (pkg.cpv, " ".join(sorted(missing_licenses)))) @@ -6048,7 +7172,7 @@ class depgraph(object): if stat.S_ISREG(st.st_mode): last_file_path = p elif stat.S_ISDIR(st.st_mode): - if os.path.basename(p) in _ignorecvs_dirs: + if os.path.basename(p) in VCS_DIRS: continue try: contents = os.listdir(p) @@ -6117,24 +7241,25 @@ class depgraph(object): if len(roots) > 1: writemsg("\nFor %s:\n" % abs_user_config, noiselevel=-1) + def _writemsg(reason, file): + writemsg(('\nThe following %s are necessary to proceed:\n' + ' (see "%s" in the portage(5) man page for more details)\n') + % (colorize('BAD', reason), file), noiselevel=-1) + if root in unstable_keyword_msg: - writemsg("\nThe following " + colorize("BAD", "keyword changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('keyword changes', 'package.accept_keywords') writemsg(format_msg(unstable_keyword_msg[root]), noiselevel=-1) if root in p_mask_change_msg: - writemsg("\nThe following " + colorize("BAD", "mask changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('mask changes', 'package.unmask') writemsg(format_msg(p_mask_change_msg[root]), noiselevel=-1) if root in use_changes_msg: - writemsg("\nThe following " + colorize("BAD", "USE changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('USE changes', 'package.use') writemsg(format_msg(use_changes_msg[root]), noiselevel=-1) if root in license_msg: - writemsg("\nThe following " + colorize("BAD", "license changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('license changes', 'package.license') writemsg(format_msg(license_msg[root]), noiselevel=-1) protect_obj = {} @@ -6148,11 +7273,12 @@ class depgraph(object): def write_changes(root, changes, file_to_write_to): file_contents = None try: - file_contents = io.open( + with io.open( _unicode_encode(file_to_write_to, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], - errors='replace').readlines() + errors='replace') as f: + file_contents = f.readlines() except IOError as e: if e.errno == errno.ENOENT: file_contents = [] @@ -6218,10 +7344,16 @@ class depgraph(object): noiselevel=-1) writemsg("".join(problems), noiselevel=-1) elif write_to_file and roots: - writemsg("\nAutounmask changes successfully written. Remember to run dispatch-conf.\n", \ + writemsg("\nAutounmask changes successfully written.\n", noiselevel=-1) + for root in roots: + chk_updated_cfg_files(root, + [os.path.join(os.sep, USER_CONFIG_PATH)]) elif not pretend and not autounmask_write and roots: - writemsg("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ + writemsg("\nUse --autounmask-write to write changes to config files (honoring\n" + "CONFIG_PROTECT). Carefully examine the list of proposed changes,\n" + "paying special attention to mask or keyword changes that may expose\n" + "experimental or unstable packages.\n", noiselevel=-1) @@ -6238,21 +7370,33 @@ class depgraph(object): self._show_circular_deps( self._dynamic_config._circular_deps_for_display) - # The slot conflict display has better noise reduction than - # the unsatisfied blockers display, so skip unsatisfied blockers - # display if there are slot conflicts (see bug #385391). + unresolved_conflicts = False if self._dynamic_config._slot_collision_info: + unresolved_conflicts = True self._show_slot_collision_notice() - elif self._dynamic_config._unsatisfied_blockers_for_display is not None: + if self._dynamic_config._unsatisfied_blockers_for_display is not None: + unresolved_conflicts = True self._show_unsatisfied_blockers( self._dynamic_config._unsatisfied_blockers_for_display) - else: + + # Only show missed updates if there are no unresolved conflicts, + # since they may be irrelevant after the conflicts are solved. + if not unresolved_conflicts: self._show_missed_update() + self._compute_abi_rebuild_info() + self._show_abi_rebuild_info() + self._show_ignored_binaries() self._display_autounmask() + for depgraph_sets in self._dynamic_config.sets.values(): + for pset in depgraph_sets.sets.values(): + for error_msg in pset.errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + # TODO: Add generic support for "set problem" handlers so that # the below warnings aren't special cases for world only. @@ -6328,7 +7472,7 @@ class depgraph(object): pkgsettings = self._frozen_config.pkgsettings[pkg.root] mreasons = get_masking_status(pkg, pkgsettings, root_config, use=self._pkg_use_enabled(pkg)) masked_packages.append((root_config, pkgsettings, - pkg.cpv, pkg.repo, pkg.metadata, mreasons)) + pkg.cpv, pkg.repo, pkg._metadata, mreasons)) if masked_packages: writemsg("\n" + colorize("BAD", "!!!") + \ " The following updates are masked by LICENSE changes:\n", @@ -6343,7 +7487,7 @@ class depgraph(object): pkgsettings = self._frozen_config.pkgsettings[pkg.root] mreasons = get_masking_status(pkg, pkgsettings, root_config, use=self._pkg_use_enabled) masked_packages.append((root_config, pkgsettings, - pkg.cpv, pkg.repo, pkg.metadata, mreasons)) + pkg.cpv, pkg.repo, pkg._metadata, mreasons)) if masked_packages: writemsg("\n" + colorize("BAD", "!!!") + \ " The following installed packages are masked:\n", @@ -6353,7 +7497,15 @@ class depgraph(object): writemsg("\n", noiselevel=-1) for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: - self._show_unsatisfied_dep(*pargs, **kwargs) + self._show_unsatisfied_dep(*pargs, + **portage._native_kwargs(kwargs)) + + if self._dynamic_config._buildpkgonly_deps_unsatisfied: + self._show_merge_list() + writemsg("\n!!! --buildpkgonly requires all " + "dependencies to be merged.\n", noiselevel=-1) + writemsg("!!! Cannot merge requested packages. " + "Merge deps and try again.\n\n", noiselevel=-1) def saveNomergeFavorites(self): """Find atoms in favorites that are not in the mergelist and add them @@ -6401,6 +7553,9 @@ class depgraph(object): continue if arg.root_config.root != root_config.root: continue + if arg.internal: + # __auto_* sets + continue k = arg.name if k in ("selected", "world") or \ not root_config.sets[k].world_candidate: @@ -6411,12 +7566,31 @@ class depgraph(object): all_added.append(SETPREFIX + k) all_added.extend(added_favorites) all_added.sort() - for a in all_added: - writemsg_stdout( - ">>> Recording %s in \"world\" favorites file...\n" % \ - colorize("INFORM", str(a)), noiselevel=-1) if all_added: - world_set.update(all_added) + skip = False + if "--ask" in self._frozen_config.myopts: + writemsg_stdout("\n", noiselevel=-1) + for a in all_added: + writemsg_stdout(" %s %s\n" % (colorize("GOOD", "*"), a), + noiselevel=-1) + writemsg_stdout("\n", noiselevel=-1) + prompt = "Would you like to add these packages to your world " \ + "favorites?" + enter_invalid = '--ask-enter-invalid' in \ + self._frozen_config.myopts + if userquery(prompt, enter_invalid) == "No": + skip = True + + if not skip: + for a in all_added: + if a.startswith(SETPREFIX): + filename = "world_sets" + else: + filename = "world" + writemsg_stdout( + ">>> Recording %s in \"%s\" favorites file...\n" % + (colorize("INFORM", _unicode(a)), filename), noiselevel=-1) + world_set.update(all_added) if world_locked: world_set.unlock() @@ -6691,14 +7865,15 @@ class depgraph(object): try: for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: self._show_unsatisfied_dep( - *pargs, check_autounmask_breakage=True, **kwargs) + *pargs, check_autounmask_breakage=True, + **portage._native_kwargs(kwargs)) except self._autounmask_breakage: return True return False def get_backtrack_infos(self): return self._dynamic_config._backtrack_infos - + class _dep_check_composite_db(dbapi): """ @@ -6793,13 +7968,8 @@ class _dep_check_composite_db(dbapi): return ret[:] def _visible(self, pkg): - if pkg.installed and "selective" not in self._depgraph._dynamic_config.myparams: - try: - arg = next(self._depgraph._iter_atoms_for_pkg(pkg)) - except (StopIteration, portage.exception.InvalidDependString): - arg = None - if arg: - return False + if pkg.installed and not self._depgraph._want_installed_pkg(pkg): + return False if pkg.installed and \ (pkg.masks or not self._depgraph._pkg_visibility_check(pkg)): # Account for packages with masks (like KEYWORDS masks) @@ -6815,24 +7985,8 @@ class _dep_check_composite_db(dbapi): if not avoid_update: if not use_ebuild_visibility and usepkgonly: return False - else: - try: - pkg_eb = self._depgraph._pkg( - pkg.cpv, "ebuild", pkg.root_config, - myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._depgraph._iter_match_pkgs( - pkg.root_config, "ebuild", - Atom("=%s" % (pkg.cpv,))): - if self._depgraph._pkg_visibility_check(pkg_eb): - pkg_eb_visible = True - break - if not pkg_eb_visible: - return False - else: - if not self._depgraph._pkg_visibility_check(pkg_eb): - return False + elif not self._depgraph._equiv_ebuild_visible(pkg): + return False in_graph = self._depgraph._dynamic_config._slot_pkg_map[ self._root].get(pkg.slot_atom) @@ -6854,7 +8008,7 @@ class _dep_check_composite_db(dbapi): return True def aux_get(self, cpv, wants): - metadata = self._cpv_pkg_map[cpv].metadata + metadata = self._cpv_pkg_map[cpv]._metadata return [metadata.get(x, "") for x in wants] def match_pkgs(self, atom): @@ -6928,14 +8082,14 @@ def _spinner_stop(spinner): portage.writemsg_stdout("... done!\n") -def backtrack_depgraph(settings, trees, myopts, myparams, +def backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, spinner): """ Raises PackageSetNotFound if myfiles contains a missing package set. """ _spinner_start(spinner, myopts) try: - return _backtrack_depgraph(settings, trees, myopts, myparams, + return _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, spinner) finally: _spinner_stop(spinner) @@ -7032,7 +8186,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): skip_masked = True skip_unsatisfied = True mergelist = mtimedb["resume"]["mergelist"] - dropped_tasks = set() + dropped_tasks = {} frozen_config = _frozen_depgraph_config(settings, trees, myopts, spinner) while True: @@ -7046,12 +8200,21 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): raise graph = mydepgraph._dynamic_config.digraph - unsatisfied_parents = dict((dep.parent, dep.parent) \ - for dep in e.value) + unsatisfied_parents = {} traversed_nodes = set() - unsatisfied_stack = list(unsatisfied_parents) + unsatisfied_stack = [(dep.parent, dep.atom) for dep in e.value] while unsatisfied_stack: - pkg = unsatisfied_stack.pop() + pkg, atom = unsatisfied_stack.pop() + if atom is not None and \ + mydepgraph._select_pkg_from_installed( + pkg.root, atom)[0] is not None: + continue + atoms = unsatisfied_parents.get(pkg) + if atoms is None: + atoms = [] + unsatisfied_parents[pkg] = atoms + if atom is not None: + atoms.append(atom) if pkg in traversed_nodes: continue traversed_nodes.add(pkg) @@ -7060,7 +8223,8 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # package scheduled for merge, removing this # package may cause the the parent package's # dependency to become unsatisfied. - for parent_node in graph.parent_nodes(pkg): + for parent_node, atom in \ + mydepgraph._dynamic_config._parent_atoms.get(pkg, []): if not isinstance(parent_node, Package) \ or parent_node.operation not in ("merge", "nomerge"): continue @@ -7068,8 +8232,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # ensure that a package with an unsatisfied depenedency # won't get pulled in, even indirectly via a soft # dependency. - unsatisfied_parents[parent_node] = parent_node - unsatisfied_stack.append(parent_node) + unsatisfied_stack.append((parent_node, atom)) unsatisfied_tuples = frozenset(tuple(parent_node) for parent_node in unsatisfied_parents @@ -7090,8 +8253,8 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # Exclude installed packages that have been removed from the graph due # to failure to build/install runtime dependencies after the dependent # package has already been installed. - dropped_tasks.update(pkg for pkg in \ - unsatisfied_parents if pkg.operation != "nomerge") + dropped_tasks.update((pkg, atoms) for pkg, atoms in \ + unsatisfied_parents.items() if pkg.operation != "nomerge") del e, graph, traversed_nodes, \ unsatisfied_parents, unsatisfied_stack @@ -7177,9 +8340,11 @@ def show_masked_packages(masked_packages): shown_comments.add(comment) portdb = root_config.trees["porttree"].dbapi for l in missing_licenses: - l_path = portdb.findLicensePath(l) if l in shown_licenses: continue + l_path = portdb.findLicensePath(l) + if l_path is None: + continue msg = ("A copy of the '%s' license" + \ " is located at '%s'.\n\n") % (l, l_path) writemsg(msg, noiselevel=-1) @@ -7206,9 +8371,9 @@ def _get_masking_status(pkg, pkgsettings, root_config, myrepo=None, use=None): portdb=root_config.trees["porttree"].dbapi, myrepo=myrepo) if not pkg.installed: - if not pkgsettings._accept_chost(pkg.cpv, pkg.metadata): + if not pkgsettings._accept_chost(pkg.cpv, pkg._metadata): mreasons.append(_MaskReason("CHOST", "CHOST: %s" % \ - pkg.metadata["CHOST"])) + pkg._metadata["CHOST"])) if pkg.invalid: for msgs in pkg.invalid.values(): @@ -7216,7 +8381,7 @@ def _get_masking_status(pkg, pkgsettings, root_config, myrepo=None, use=None): mreasons.append( _MaskReason("invalid", "invalid: %s" % (msg,))) - if not pkg.metadata["SLOT"]: + if not pkg._metadata["SLOT"]: mreasons.append( _MaskReason("invalid", "SLOT: undefined")) diff --git a/portage_with_autodep/pym/_emerge/depgraph.pyo b/portage_with_autodep/pym/_emerge/depgraph.pyo Binary files differindex ba00a11..db9b676 100644 --- a/portage_with_autodep/pym/_emerge/depgraph.pyo +++ b/portage_with_autodep/pym/_emerge/depgraph.pyo diff --git a/portage_with_autodep/pym/_emerge/emergelog.py b/portage_with_autodep/pym/_emerge/emergelog.py index b1b093f..aea94f7 100644 --- a/portage_with_autodep/pym/_emerge/emergelog.py +++ b/portage_with_autodep/pym/_emerge/emergelog.py @@ -1,7 +1,7 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import unicode_literals import io import sys @@ -20,10 +20,6 @@ from portage.output import xtermTitle _disable = True _emerge_log_dir = '/var/log' -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_log_fmt = _unicode_decode("%.0f: %s\n") - def emergelog(xterm_titles, mystr, short_msg=None): if _disable: @@ -51,10 +47,10 @@ def emergelog(xterm_titles, mystr, short_msg=None): mode=0o660) mylock = portage.locks.lockfile(file_path) try: - mylogfile.write(_log_fmt % (time.time(), mystr)) + mylogfile.write("%.0f: %s\n" % (time.time(), mystr)) mylogfile.close() finally: portage.locks.unlockfile(mylock) except (IOError,OSError,portage.exception.PortageException) as e: if secpass >= 1: - print("emergelog():",e, file=sys.stderr) + portage.util.writemsg("emergelog(): %s\n" % (e,), noiselevel=-1) diff --git a/portage_with_autodep/pym/_emerge/emergelog.pyo b/portage_with_autodep/pym/_emerge/emergelog.pyo Binary files differindex 7e67bd3..997a1c0 100644 --- a/portage_with_autodep/pym/_emerge/emergelog.pyo +++ b/portage_with_autodep/pym/_emerge/emergelog.pyo diff --git a/portage_with_autodep/pym/_emerge/getloadavg.py b/portage_with_autodep/pym/_emerge/getloadavg.py index e9babf1..6a2794f 100644 --- a/portage_with_autodep/pym/_emerge/getloadavg.py +++ b/portage_with_autodep/pym/_emerge/getloadavg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -11,7 +11,8 @@ if getloadavg is None: Raises OSError if the load average was unobtainable. """ try: - loadavg_str = open('/proc/loadavg').readline() + with open('/proc/loadavg') as f: + loadavg_str = f.readline() except IOError: # getloadavg() is only supposed to raise OSError, so convert raise OSError('unknown') diff --git a/portage_with_autodep/pym/_emerge/getloadavg.pyo b/portage_with_autodep/pym/_emerge/getloadavg.pyo Binary files differindex 56bda8c..c6c99db 100644 --- a/portage_with_autodep/pym/_emerge/getloadavg.pyo +++ b/portage_with_autodep/pym/_emerge/getloadavg.pyo diff --git a/portage_with_autodep/pym/_emerge/help.py b/portage_with_autodep/pym/_emerge/help.py index a1dbb37..52cfd00 100644 --- a/portage_with_autodep/pym/_emerge/help.py +++ b/portage_with_autodep/pym/_emerge/help.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -9,11 +9,11 @@ def help(): print(bold("emerge:")+" the other white meat (command-line interface to the Portage system)") print(bold("Usage:")) print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] [ "+turquoise("ebuild")+" | "+turquoise("tbz2")+" | "+turquoise("file")+" | "+turquoise("@set")+" | "+turquoise("atom")+" ] [ ... ]") - print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("system")+" | "+turquoise("world")+" >") + print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("@system")+" | "+turquoise("@world")+" >") print(" "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >") print(" "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]") - print(" "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("--verbose")+" ] ") - print(bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpPqrsStuvV")+"]") + print(" "+turquoise("emerge")+" "+turquoise("--help")) + print(bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpPqrsStuvVw")+"]") print(" [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]") print(" [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]") print(" [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]") diff --git a/portage_with_autodep/pym/_emerge/help.pyo b/portage_with_autodep/pym/_emerge/help.pyo Binary files differindex f6fea4e..b309ff2 100644 --- a/portage_with_autodep/pym/_emerge/help.pyo +++ b/portage_with_autodep/pym/_emerge/help.pyo diff --git a/portage_with_autodep/pym/_emerge/is_valid_package_atom.py b/portage_with_autodep/pym/_emerge/is_valid_package_atom.py index 7cb2a5b..112afc1 100644 --- a/portage_with_autodep/pym/_emerge/is_valid_package_atom.py +++ b/portage_with_autodep/pym/_emerge/is_valid_package_atom.py @@ -1,11 +1,12 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import re from portage.dep import isvalidatom def insert_category_into_atom(atom, category): - alphanum = re.search(r'\w', atom) + # Handle '*' character for "extended syntax" wildcard support. + alphanum = re.search(r'[\*\w]', atom, re.UNICODE) if alphanum: ret = atom[:alphanum.start()] + "%s/" % category + \ atom[alphanum.start():] @@ -14,7 +15,7 @@ def insert_category_into_atom(atom, category): return ret def is_valid_package_atom(x, allow_repo=False): - if "/" not in x: + if "/" not in x.split(":")[0]: x2 = insert_category_into_atom(x, 'cat') if x2 != None: x = x2 diff --git a/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo Binary files differindex 20edc85..68aaa52 100644 --- a/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo +++ b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/main.py b/portage_with_autodep/pym/_emerge/main.py index c52a3ea..89413a9 100644 --- a/portage_with_autodep/pym/_emerge/main.py +++ b/portage_with_autodep/pym/_emerge/main.py @@ -1,52 +1,24 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -import logging -import signal -import stat -import subprocess -import sys -import textwrap import platform +import sys + import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.news:count_unread_news,display_news_notifications', + 'logging', + 'portage.dep:Atom', + 'portage.util:writemsg_level', + 'textwrap', + '_emerge.actions:load_emerge_config,run_action,' + \ + 'validate_ebuild_environment', + '_emerge.help:help@emerge_help', + '_emerge.is_valid_package_atom:insert_category_into_atom' ) from portage import os -from portage import _encodings -from portage import _unicode_decode -import _emerge.help -import portage.xpak, errno, re, time -from portage.output import colorize, xtermTitle, xtermTitleReset -from portage.output import create_color_func -good = create_color_func("GOOD") -bad = create_color_func("BAD") - -from portage.const import _ENABLE_DYN_LINK_MAP -import portage.elog -import portage.util -import portage.locks -import portage.exception -from portage.data import secpass -from portage.dbapi.dep_expand import dep_expand -from portage.util import normalize_path as normpath -from portage.util import (shlex_split, varexpand, - writemsg_level, writemsg_stdout) -from portage._sets import SETPREFIX -from portage._global_updates import _global_updates - -from _emerge.actions import action_config, action_sync, action_metadata, \ - action_regen, action_search, action_uninstall, action_info, action_build, \ - adjust_configs, chk_updated_cfg_files, display_missing_pkg_set, \ - display_news_notification, getportageversion, load_emerge_config -import _emerge -from _emerge.emergelog import emergelog -from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo -from _emerge.is_valid_package_atom import is_valid_package_atom -from _emerge.stdout_spinner import stdout_spinner -from _emerge.userquery import userquery +from portage.util._argparse import ArgumentParser if sys.hexversion >= 0x3000000: long = int @@ -60,6 +32,7 @@ options=[ "--debug", "--digest", "--emptytree", +"--verbose-conflicts", "--fetchonly", "--fetch-all-uri", "--ignore-default-opts", "--noconfmem", @@ -75,7 +48,6 @@ options=[ "--tree", "--unordered-display", "--update", -"--verbose", "--verbose-main-repo-display", ] @@ -96,7 +68,7 @@ shortmapping={ "s":"--search", "S":"--searchdesc", "t":"--tree", "u":"--update", -"v":"--verbose", "V":"--version" +"V":"--version" } COWSAY_MOO = """ @@ -114,325 +86,6 @@ COWSAY_MOO = """ """ -def chk_updated_info_files(root, infodirs, prev_mtimes, retval): - - if os.path.exists("/usr/bin/install-info"): - out = portage.output.EOutput() - regen_infodirs=[] - for z in infodirs: - if z=='': - continue - inforoot=normpath(root+z) - if os.path.isdir(inforoot) and \ - not [x for x in os.listdir(inforoot) \ - if x.startswith('.keepinfodir')]: - infomtime = os.stat(inforoot)[stat.ST_MTIME] - if inforoot not in prev_mtimes or \ - prev_mtimes[inforoot] != infomtime: - regen_infodirs.append(inforoot) - - if not regen_infodirs: - portage.writemsg_stdout("\n") - if portage.util.noiselimit >= 0: - out.einfo("GNU info directory index is up-to-date.") - else: - portage.writemsg_stdout("\n") - if portage.util.noiselimit >= 0: - out.einfo("Regenerating GNU info directory index...") - - dir_extensions = ("", ".gz", ".bz2") - icount=0 - badcount=0 - errmsg = "" - for inforoot in regen_infodirs: - if inforoot=='': - continue - - if not os.path.isdir(inforoot) or \ - not os.access(inforoot, os.W_OK): - continue - - file_list = os.listdir(inforoot) - file_list.sort() - dir_file = os.path.join(inforoot, "dir") - moved_old_dir = False - processed_count = 0 - for x in file_list: - if x.startswith(".") or \ - os.path.isdir(os.path.join(inforoot, x)): - continue - if x.startswith("dir"): - skip = False - for ext in dir_extensions: - if x == "dir" + ext or \ - x == "dir" + ext + ".old": - skip = True - break - if skip: - continue - if processed_count == 0: - for ext in dir_extensions: - try: - os.rename(dir_file + ext, dir_file + ext + ".old") - moved_old_dir = True - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - processed_count += 1 - try: - proc = subprocess.Popen( - ['/usr/bin/install-info', - '--dir-file=%s' % os.path.join(inforoot, "dir"), - os.path.join(inforoot, x)], - env=dict(os.environ, LANG="C", LANGUAGE="C"), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - except OSError: - myso = None - else: - myso = _unicode_decode( - proc.communicate()[0]).rstrip("\n") - proc.wait() - existsstr="already exists, for file `" - if myso: - if re.search(existsstr,myso): - # Already exists... Don't increment the count for this. - pass - elif myso[:44]=="install-info: warning: no info dir entry in ": - # This info file doesn't contain a DIR-header: install-info produces this - # (harmless) warning (the --quiet switch doesn't seem to work). - # Don't increment the count for this. - pass - else: - badcount=badcount+1 - errmsg += myso + "\n" - icount=icount+1 - - if moved_old_dir and not os.path.exists(dir_file): - # We didn't generate a new dir file, so put the old file - # back where it was originally found. - for ext in dir_extensions: - try: - os.rename(dir_file + ext + ".old", dir_file + ext) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - - # Clean dir.old cruft so that they don't prevent - # unmerge of otherwise empty directories. - for ext in dir_extensions: - try: - os.unlink(dir_file + ext + ".old") - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - - #update mtime so we can potentially avoid regenerating. - prev_mtimes[inforoot] = os.stat(inforoot)[stat.ST_MTIME] - - if badcount: - out.eerror("Processed %d info files; %d errors." % \ - (icount, badcount)) - writemsg_level(errmsg, level=logging.ERROR, noiselevel=-1) - else: - if icount > 0 and portage.util.noiselimit >= 0: - out.einfo("Processed %d info files." % (icount,)) - -def display_preserved_libs(vardbapi, myopts): - MAX_DISPLAY = 3 - - if vardbapi._linkmap is None or \ - vardbapi._plib_registry is None: - # preserve-libs is entirely disabled - return - - # Explicitly load and prune the PreservedLibsRegistry in order - # to ensure that we do not display stale data. - vardbapi._plib_registry.load() - - if vardbapi._plib_registry.hasEntries(): - if "--quiet" in myopts: - print() - print(colorize("WARN", "!!!") + " existing preserved libs found") - return - else: - print() - print(colorize("WARN", "!!!") + " existing preserved libs:") - - plibdata = vardbapi._plib_registry.getPreservedLibs() - linkmap = vardbapi._linkmap - consumer_map = {} - owners = {} - - try: - linkmap.rebuild() - except portage.exception.CommandNotFound as e: - writemsg_level("!!! Command Not Found: %s\n" % (e,), - level=logging.ERROR, noiselevel=-1) - del e - else: - search_for_owners = set() - for cpv in plibdata: - internal_plib_keys = set(linkmap._obj_key(f) \ - for f in plibdata[cpv]) - for f in plibdata[cpv]: - if f in consumer_map: - continue - consumers = [] - for c in linkmap.findConsumers(f): - # Filter out any consumers that are also preserved libs - # belonging to the same package as the provider. - if linkmap._obj_key(c) not in internal_plib_keys: - consumers.append(c) - consumers.sort() - consumer_map[f] = consumers - search_for_owners.update(consumers[:MAX_DISPLAY+1]) - - owners = {} - for f in search_for_owners: - owner_set = set() - for owner in linkmap.getOwners(f): - owner_dblink = vardbapi._dblink(owner) - if owner_dblink.exists(): - owner_set.add(owner_dblink) - if owner_set: - owners[f] = owner_set - - for cpv in plibdata: - print(colorize("WARN", ">>>") + " package: %s" % cpv) - samefile_map = {} - for f in plibdata[cpv]: - obj_key = linkmap._obj_key(f) - alt_paths = samefile_map.get(obj_key) - if alt_paths is None: - alt_paths = set() - samefile_map[obj_key] = alt_paths - alt_paths.add(f) - - for alt_paths in samefile_map.values(): - alt_paths = sorted(alt_paths) - for p in alt_paths: - print(colorize("WARN", " * ") + " - %s" % (p,)) - f = alt_paths[0] - consumers = consumer_map.get(f, []) - for c in consumers[:MAX_DISPLAY]: - print(colorize("WARN", " * ") + " used by %s (%s)" % \ - (c, ", ".join(x.mycpv for x in owners.get(c, [])))) - if len(consumers) == MAX_DISPLAY + 1: - print(colorize("WARN", " * ") + " used by %s (%s)" % \ - (consumers[MAX_DISPLAY], ", ".join(x.mycpv \ - for x in owners.get(consumers[MAX_DISPLAY], [])))) - elif len(consumers) > MAX_DISPLAY: - print(colorize("WARN", " * ") + " used by %d other files" % (len(consumers) - MAX_DISPLAY)) - print("Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries") - -def post_emerge(myaction, myopts, myfiles, - target_root, trees, mtimedb, retval): - """ - Misc. things to run at the end of a merge session. - - Update Info Files - Update Config Files - Update News Items - Commit mtimeDB - Display preserved libs warnings - - @param myaction: The action returned from parse_opts() - @type myaction: String - @param myopts: emerge options - @type myopts: dict - @param myfiles: emerge arguments - @type myfiles: list - @param target_root: The target EROOT for myaction - @type target_root: String - @param trees: A dictionary mapping each ROOT to it's package databases - @type trees: dict - @param mtimedb: The mtimeDB to store data needed across merge invocations - @type mtimedb: MtimeDB class instance - @param retval: Emerge's return value - @type retval: Int - """ - - root_config = trees[target_root]["root_config"] - vardbapi = trees[target_root]['vartree'].dbapi - settings = vardbapi.settings - info_mtimes = mtimedb["info"] - - # Load the most current variables from ${ROOT}/etc/profile.env - settings.unlock() - settings.reload() - settings.regenerate() - settings.lock() - - config_protect = shlex_split(settings.get("CONFIG_PROTECT", "")) - infodirs = settings.get("INFOPATH","").split(":") + \ - settings.get("INFODIR","").split(":") - - os.chdir("/") - - if retval == os.EX_OK: - exit_msg = " *** exiting successfully." - else: - exit_msg = " *** exiting unsuccessfully with status '%s'." % retval - emergelog("notitles" not in settings.features, exit_msg) - - _flush_elog_mod_echo() - - if not vardbapi._pkgs_changed: - # GLEP 42 says to display news *after* an emerge --pretend - if "--pretend" in myopts: - display_news_notification(root_config, myopts) - # If vdb state has not changed then there's nothing else to do. - return - - vdb_path = os.path.join(root_config.settings['EROOT'], portage.VDB_PATH) - portage.util.ensure_dirs(vdb_path) - vdb_lock = None - if os.access(vdb_path, os.W_OK) and not "--pretend" in myopts: - vardbapi.lock() - vdb_lock = True - - if vdb_lock: - try: - if "noinfo" not in settings.features: - chk_updated_info_files(target_root, - infodirs, info_mtimes, retval) - mtimedb.commit() - finally: - if vdb_lock: - vardbapi.unlock() - - display_preserved_libs(vardbapi, myopts) - chk_updated_cfg_files(settings['EROOT'], config_protect) - - display_news_notification(root_config, myopts) - - postemerge = os.path.join(settings["PORTAGE_CONFIGROOT"], - portage.USER_CONFIG_PATH, "bin", "post_emerge") - if os.access(postemerge, os.X_OK): - hook_retval = portage.process.spawn( - [postemerge], env=settings.environ()) - if hook_retval != os.EX_OK: - writemsg_level( - " %s spawn failed of %s\n" % (bad("*"), postemerge,), - level=logging.ERROR, noiselevel=-1) - - clean_logs(settings) - - if "--quiet" not in myopts and \ - myaction is None and "@world" in myfiles: - show_depclean_suggestion() - -def show_depclean_suggestion(): - out = portage.output.EOutput() - msg = "After world updates, it is important to remove " + \ - "obsolete packages with emerge --depclean. Refer " + \ - "to `man emerge` for more information." - for line in textwrap.wrap(msg, 72): - out.ewarn(line) - def multiple_actions(action1, action2): sys.stderr.write("\n!!! Multiple actions requested... Please choose one only.\n") sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2)) @@ -454,6 +107,16 @@ def insert_optional_args(args): return False valid_integers = valid_integers() + + class valid_floats(object): + def __contains__(self, s): + try: + return float(s) >= 0 + except (ValueError, OverflowError): + return False + + valid_floats = valid_floats() + y_or_n = ('y', 'n',) new_args = [] @@ -467,6 +130,7 @@ def insert_optional_args(args): '--buildpkg' : y_or_n, '--complete-graph' : y_or_n, '--deep' : valid_integers, + '--depclean-lib-check' : y_or_n, '--deselect' : y_or_n, '--binpkg-respect-use' : y_or_n, '--fail-clean' : y_or_n, @@ -474,9 +138,12 @@ def insert_optional_args(args): '--getbinpkgonly' : y_or_n, '--jobs' : valid_integers, '--keep-going' : y_or_n, + '--load-average' : valid_floats, '--package-moves' : y_or_n, '--quiet' : y_or_n, '--quiet-build' : y_or_n, + '--quiet-fail' : y_or_n, + '--rebuild-if-new-slot': y_or_n, '--rebuild-if-new-rev' : y_or_n, '--rebuild-if-new-ver' : y_or_n, '--rebuild-if-unbuilt' : y_or_n, @@ -487,11 +154,9 @@ def insert_optional_args(args): "--use-ebuild-visibility": y_or_n, '--usepkg' : y_or_n, '--usepkgonly' : y_or_n, + '--verbose' : y_or_n, } - if _ENABLE_DYN_LINK_MAP: - default_arg_opts['--depclean-lib-check'] = y_or_n - short_arg_opts = { 'D' : valid_integers, 'j' : valid_integers, @@ -507,6 +172,8 @@ def insert_optional_args(args): 'k' : y_or_n, 'K' : y_or_n, 'q' : y_or_n, + 'v' : y_or_n, + 'w' : y_or_n, } arg_stack = args[:] @@ -595,14 +262,17 @@ def _find_bad_atoms(atoms, less_strict=False): """ bad_atoms = [] for x in ' '.join(atoms).split(): + atom = x + if "/" not in x.split(":")[0]: + x_cat = insert_category_into_atom(x, 'dummy-category') + if x_cat is not None: + atom = x_cat + bad_atom = False try: - atom = portage.dep.Atom(x, allow_wildcard=True, allow_repo=less_strict) + atom = Atom(atom, allow_wildcard=True, allow_repo=less_strict) except portage.exception.InvalidAtom: - try: - atom = portage.dep.Atom("*/"+x, allow_wildcard=True, allow_repo=less_strict) - except portage.exception.InvalidAtom: - bad_atom = True + bad_atom = True if bad_atom or (atom.operator and not less_strict) or atom.blocker or atom.use: bad_atoms.append(x) @@ -630,31 +300,26 @@ def parse_opts(tmpcmdline, silent=False): "--ask": { "shortopt" : "-a", "help" : "prompt before performing any actions", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask": { "help" : "automatically unmask packages", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-unrestricted-atoms": { "help" : "write autounmask changes with >= atoms if possible", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-keep-masks": { "help" : "don't add package.unmask entries", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-write": { "help" : "write changes made by --autounmask to disk", - "type" : "choice", "choices" : true_y_or_n }, @@ -663,6 +328,11 @@ def parse_opts(tmpcmdline, silent=False): "action":"store" }, + "--accept-restrict": { + "help":"temporarily override ACCEPT_RESTRICT", + "action":"store" + }, + "--backtrack": { "help" : "Specifies how many times to backtrack if dependency " + \ @@ -674,7 +344,6 @@ def parse_opts(tmpcmdline, silent=False): "--buildpkg": { "shortopt" : "-b", "help" : "build binary packages", - "type" : "choice", "choices" : true_y_or_n }, @@ -692,19 +361,21 @@ def parse_opts(tmpcmdline, silent=False): }, "--color": { "help":"enable or disable color output", - "type":"choice", "choices":("y", "n") }, "--complete-graph": { "help" : "completely account for all known dependencies", - "type" : "choice", "choices" : true_y_or_n }, + "--complete-graph-if-new-use": { + "help" : "trigger --complete-graph behavior if USE or IUSE will change for an installed package", + "choices" : y_or_n + }, + "--complete-graph-if-new-ver": { "help" : "trigger --complete-graph behavior if an installed package version will change (upgrade or downgrade)", - "type" : "choice", "choices" : y_or_n }, @@ -720,15 +391,18 @@ def parse_opts(tmpcmdline, silent=False): "action" : "store" }, + "--depclean-lib-check": { + "help" : "check for consumers of libraries before removing them", + "choices" : true_y_or_n + }, + "--deselect": { "help" : "remove atoms/sets from the world file", - "type" : "choice", "choices" : true_y_or_n }, "--dynamic-deps": { "help": "substitute the dependencies of installed packages with the dependencies of unbuilt ebuilds", - "type": "choice", "choices": y_or_n }, @@ -742,10 +416,18 @@ def parse_opts(tmpcmdline, silent=False): "--fail-clean": { "help" : "clean temp files after build failure", - "type" : "choice", "choices" : true_y_or_n }, + "--ignore-built-slot-operator-deps": { + "help": "Ignore the slot/sub-slot := operator parts of dependencies that have " + "been recorded when packages where built. This option is intended " + "only for debugging purposes, and it only affects built packages " + "that specify slot/sub-slot := operator dependencies using the " + "experimental \"4-slot-abi\" EAPI.", + "choices": y_or_n + }, + "--jobs": { "shortopt" : "-j", @@ -758,7 +440,6 @@ def parse_opts(tmpcmdline, silent=False): "--keep-going": { "help" : "continue as much as possible after an error", - "type" : "choice", "choices" : true_y_or_n }, @@ -773,18 +454,15 @@ def parse_opts(tmpcmdline, silent=False): "--misspell-suggestions": { "help" : "enable package name misspell suggestions", - "type" : "choice", "choices" : ("y", "n") }, "--with-bdeps": { "help":"include unnecessary build time dependencies", - "type":"choice", "choices":("y", "n") }, "--reinstall": { "help":"specify conditions to trigger package reinstallation", - "type":"choice", "choices":["changed-use"] }, @@ -799,21 +477,18 @@ def parse_opts(tmpcmdline, silent=False): "--binpkg-respect-use": { "help" : "discard binary packages if their use flags \ don't match the current configuration", - "type" : "choice", "choices" : true_y_or_n }, "--getbinpkg": { "shortopt" : "-g", "help" : "fetch binary packages", - "type" : "choice", "choices" : true_y_or_n }, "--getbinpkgonly": { "shortopt" : "-G", "help" : "fetch binary packages only", - "type" : "choice", "choices" : true_y_or_n }, @@ -842,29 +517,48 @@ def parse_opts(tmpcmdline, silent=False): "--package-moves": { "help" : "perform package moves when necessary", - "type" : "choice", "choices" : true_y_or_n }, + "--prefix": { + "help" : "specify the installation prefix", + "action" : "store" + }, + + "--pkg-format": { + "help" : "format of result binary package", + "action" : "store", + }, + "--quiet": { "shortopt" : "-q", "help" : "reduced or condensed output", - "type" : "choice", "choices" : true_y_or_n }, "--quiet-build": { "help" : "redirect build output to logs", - "type" : "choice", "choices" : true_y_or_n, }, + "--quiet-fail": { + "help" : "suppresses display of the build log on stdout", + "choices" : true_y_or_n, + }, + + "--rebuild-if-new-slot": { + "help" : ("Automatically rebuild or reinstall packages when slot/sub-slot := " + "operator dependencies can be satisfied by a newer slot, so that " + "older packages slots will become eligible for removal by the " + "--depclean action as soon as possible."), + "choices" : true_y_or_n + }, + "--rebuild-if-new-rev": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built, " + \ "if the dependency is not already installed with the " + \ "same version and revision.", - "type" : "choice", "choices" : true_y_or_n }, @@ -873,21 +567,18 @@ def parse_opts(tmpcmdline, silent=False): "used at both build-time and run-time are built, " + \ "if the dependency is not already installed with the " + \ "same version. Revision numbers are ignored.", - "type" : "choice", "choices" : true_y_or_n }, "--rebuild-if-unbuilt": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built.", - "type" : "choice", "choices" : true_y_or_n }, "--rebuilt-binaries": { "help" : "replace installed packages with binary " + \ "packages that have been rebuilt", - "type" : "choice", "choices" : true_y_or_n }, @@ -904,26 +595,23 @@ def parse_opts(tmpcmdline, silent=False): "--root-deps": { "help" : "modify interpretation of depedencies", - "type" : "choice", "choices" :("True", "rdeps") }, "--select": { + "shortopt" : "-w", "help" : "add specified packages to the world set " + \ "(inverse of --oneshot)", - "type" : "choice", "choices" : true_y_or_n }, "--selective": { "help" : "identical to --noreplace", - "type" : "choice", "choices" : true_y_or_n }, "--use-ebuild-visibility": { "help" : "use unbuilt ebuild metadata for visibility checks on built packages", - "type" : "choice", "choices" : true_y_or_n }, @@ -937,42 +625,35 @@ def parse_opts(tmpcmdline, silent=False): "--usepkg": { "shortopt" : "-k", "help" : "use binary packages", - "type" : "choice", "choices" : true_y_or_n }, "--usepkgonly": { "shortopt" : "-K", "help" : "use only binary packages", - "type" : "choice", "choices" : true_y_or_n }, + "--verbose": { + "shortopt" : "-v", + "help" : "verbose output", + "choices" : true_y_or_n + }, } - if _ENABLE_DYN_LINK_MAP: - argument_options["--depclean-lib-check"] = { - "help" : "check for consumers of libraries before removing them", - "type" : "choice", - "choices" : true_y_or_n - } - - from optparse import OptionParser - parser = OptionParser() - if parser.has_option("--help"): - parser.remove_option("--help") + parser = ArgumentParser(add_help=False) for action_opt in actions: - parser.add_option("--" + action_opt, action="store_true", + parser.add_argument("--" + action_opt, action="store_true", dest=action_opt.replace("-", "_"), default=False) for myopt in options: - parser.add_option(myopt, action="store_true", + parser.add_argument(myopt, action="store_true", dest=myopt.lstrip("--").replace("-", "_"), default=False) for shortopt, longopt in shortmapping.items(): - parser.add_option("-" + shortopt, action="store_true", + parser.add_argument("-" + shortopt, action="store_true", dest=longopt.lstrip("--").replace("-", "_"), default=False) for myalias, myopt in longopt_aliases.items(): - parser.add_option(myalias, action="store_true", + parser.add_argument(myalias, action="store_true", dest=myopt.lstrip("--").replace("-", "_"), default=False) for myopt, kwargs in argument_options.items(): @@ -980,12 +661,12 @@ def parse_opts(tmpcmdline, silent=False): args = [myopt] if shortopt is not None: args.append(shortopt) - parser.add_option(dest=myopt.lstrip("--").replace("-", "_"), + parser.add_argument(dest=myopt.lstrip("--").replace("-", "_"), *args, **kwargs) tmpcmdline = insert_optional_args(tmpcmdline) - myoptions, myargs = parser.parse_args(args=tmpcmdline) + myoptions, myargs = parser.parse_known_args(args=tmpcmdline) if myoptions.ask in true_y: myoptions.ask = True @@ -1031,9 +712,8 @@ def parse_opts(tmpcmdline, silent=False): else: myoptions.complete_graph = None - if _ENABLE_DYN_LINK_MAP: - if myoptions.depclean_lib_check in true_y: - myoptions.depclean_lib_check = True + if myoptions.depclean_lib_check in true_y: + myoptions.depclean_lib_check = True if myoptions.exclude: bad_atoms = _find_bad_atoms(myoptions.exclude) @@ -1100,6 +780,12 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.quiet_build in true_y: myoptions.quiet_build = 'y' + if myoptions.quiet_fail in true_y: + myoptions.quiet_fail = 'y' + + if myoptions.rebuild_if_new_slot in true_y: + myoptions.rebuild_if_new_slot = 'y' + if myoptions.rebuild_if_new_ver in true_y: myoptions.rebuild_if_new_ver = True else: @@ -1185,6 +871,9 @@ def parse_opts(tmpcmdline, silent=False): myoptions.jobs = jobs + if myoptions.load_average == "True": + myoptions.load_average = None + if myoptions.load_average: try: load_average = float(myoptions.load_average) @@ -1229,6 +918,11 @@ def parse_opts(tmpcmdline, silent=False): else: myoptions.usepkgonly = None + if myoptions.verbose in true_y: + myoptions.verbose = True + else: + myoptions.verbose = None + for myopt in options: v = getattr(myoptions, myopt.lstrip("--").replace("-", "_")) if v: @@ -1253,320 +947,10 @@ def parse_opts(tmpcmdline, silent=False): if myaction is None and myoptions.deselect is True: myaction = 'deselect' - if myargs and isinstance(myargs[0], bytes): - for i in range(len(myargs)): - myargs[i] = portage._unicode_decode(myargs[i]) - myfiles += myargs return myaction, myopts, myfiles -# Warn about features that may confuse users and -# lead them to report invalid bugs. -_emerge_features_warn = frozenset(['keeptemp', 'keepwork']) - -def validate_ebuild_environment(trees): - features_warn = set() - for myroot in trees: - settings = trees[myroot]["vartree"].settings - settings.validate() - features_warn.update( - _emerge_features_warn.intersection(settings.features)) - - if features_warn: - msg = "WARNING: The FEATURES variable contains one " + \ - "or more values that should be disabled under " + \ - "normal circumstances: %s" % " ".join(features_warn) - out = portage.output.EOutput() - for line in textwrap.wrap(msg, 65): - out.ewarn(line) - -def apply_priorities(settings): - ionice(settings) - nice(settings) - -def nice(settings): - try: - os.nice(int(settings.get("PORTAGE_NICENESS", "0"))) - except (OSError, ValueError) as e: - out = portage.output.EOutput() - out.eerror("Failed to change nice value to '%s'" % \ - settings["PORTAGE_NICENESS"]) - out.eerror("%s\n" % str(e)) - -def ionice(settings): - - ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") - if ionice_cmd: - ionice_cmd = portage.util.shlex_split(ionice_cmd) - if not ionice_cmd: - return - - variables = {"PID" : str(os.getpid())} - cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - # The OS kernel probably doesn't support ionice, - # so return silently. - return - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") - -def clean_logs(settings): - - if "clean-logs" not in settings.features: - return - - clean_cmd = settings.get("PORT_LOGDIR_CLEAN") - if clean_cmd: - clean_cmd = shlex_split(clean_cmd) - if not clean_cmd: - return - - logdir = settings.get("PORT_LOGDIR") - if logdir is None or not os.path.isdir(logdir): - return - - variables = {"PORT_LOGDIR" : logdir} - cmd = [varexpand(x, mydict=variables) for x in clean_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - rval = 127 - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for " - "PORT_LOGDIR_CLEAN usage instructions.") - -def setconfig_fallback(root_config): - from portage._sets.base import DummyPackageSet - from portage._sets.files import WorldSelectedSet - from portage._sets.profiles import PackagesSystemSet - setconfig = root_config.setconfig - setconfig.psets['world'] = DummyPackageSet(atoms=['@selected', '@system']) - setconfig.psets['selected'] = WorldSelectedSet(root_config.settings['EROOT']) - setconfig.psets['system'] = \ - PackagesSystemSet(root_config.settings.profiles) - root_config.sets = setconfig.getSets() - -def get_missing_sets(root_config): - # emerge requires existence of "world", "selected", and "system" - missing_sets = [] - - for s in ("selected", "system", "world",): - if s not in root_config.sets: - missing_sets.append(s) - - return missing_sets - -def missing_sets_warning(root_config, missing_sets): - if len(missing_sets) > 2: - missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1]) - missing_sets_str += ', and "%s"' % missing_sets[-1] - elif len(missing_sets) == 2: - missing_sets_str = '"%s" and "%s"' % tuple(missing_sets) - else: - missing_sets_str = '"%s"' % missing_sets[-1] - msg = ["emerge: incomplete set configuration, " + \ - "missing set(s): %s" % missing_sets_str] - if root_config.sets: - msg.append(" sets defined: %s" % ", ".join(root_config.sets)) - global_config_path = portage.const.GLOBAL_CONFIG_PATH - if root_config.settings['EPREFIX']: - global_config_path = os.path.join(root_config.settings['EPREFIX'], - portage.const.GLOBAL_CONFIG_PATH.lstrip(os.sep)) - msg.append(" This usually means that '%s'" % \ - (os.path.join(global_config_path, "sets/portage.conf"),)) - msg.append(" is missing or corrupt.") - msg.append(" Falling back to default world and system set configuration!!!") - for line in msg: - writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1) - -def ensure_required_sets(trees): - warning_shown = False - for root_trees in trees.values(): - missing_sets = get_missing_sets(root_trees["root_config"]) - if missing_sets and not warning_shown: - warning_shown = True - missing_sets_warning(root_trees["root_config"], missing_sets) - if missing_sets: - setconfig_fallback(root_trees["root_config"]) - -def expand_set_arguments(myfiles, myaction, root_config): - retval = os.EX_OK - setconfig = root_config.setconfig - - sets = setconfig.getSets() - - # In order to know exactly which atoms/sets should be added to the - # world file, the depgraph performs set expansion later. It will get - # confused about where the atoms came from if it's not allowed to - # expand them itself. - do_not_expand = (None, ) - newargs = [] - for a in myfiles: - if a in ("system", "world"): - newargs.append(SETPREFIX+a) - else: - newargs.append(a) - myfiles = newargs - del newargs - newargs = [] - - # separators for set arguments - ARG_START = "{" - ARG_END = "}" - - for i in range(0, len(myfiles)): - if myfiles[i].startswith(SETPREFIX): - start = 0 - end = 0 - x = myfiles[i][len(SETPREFIX):] - newset = "" - while x: - start = x.find(ARG_START) - end = x.find(ARG_END) - if start > 0 and start < end: - namepart = x[:start] - argpart = x[start+1:end] - - # TODO: implement proper quoting - args = argpart.split(",") - options = {} - for a in args: - if "=" in a: - k, v = a.split("=", 1) - options[k] = v - else: - options[a] = "True" - setconfig.update(namepart, options) - newset += (x[:start-len(namepart)]+namepart) - x = x[end+len(ARG_END):] - else: - newset += x - x = "" - myfiles[i] = SETPREFIX+newset - - sets = setconfig.getSets() - - # display errors that occurred while loading the SetConfig instance - for e in setconfig.errors: - print(colorize("BAD", "Error during set creation: %s" % e)) - - unmerge_actions = ("unmerge", "prune", "clean", "depclean") - - for a in myfiles: - if a.startswith(SETPREFIX): - s = a[len(SETPREFIX):] - if s not in sets: - display_missing_pkg_set(root_config, s) - return (None, 1) - setconfig.active.append(s) - try: - set_atoms = setconfig.getSetAtoms(s) - except portage.exception.PackageSetNotFound as e: - writemsg_level(("emerge: the given set '%s' " + \ - "contains a non-existent set named '%s'.\n") % \ - (s, e), level=logging.ERROR, noiselevel=-1) - return (None, 1) - if myaction in unmerge_actions and \ - not sets[s].supportsOperation("unmerge"): - sys.stderr.write("emerge: the given set '%s' does " % s + \ - "not support unmerge operations\n") - retval = 1 - elif not set_atoms: - print("emerge: '%s' is an empty set" % s) - elif myaction not in do_not_expand: - newargs.extend(set_atoms) - else: - newargs.append(SETPREFIX+s) - for e in sets[s].errors: - print(e) - else: - newargs.append(a) - return (newargs, retval) - -def repo_name_check(trees): - missing_repo_names = set() - for root_trees in trees.values(): - porttree = root_trees.get("porttree") - if porttree: - portdb = porttree.dbapi - missing_repo_names.update(portdb.getMissingRepoNames()) - if portdb.porttree_root in missing_repo_names and \ - not os.path.exists(os.path.join( - portdb.porttree_root, "profiles")): - # This is normal if $PORTDIR happens to be empty, - # so don't warn about it. - missing_repo_names.remove(portdb.porttree_root) - - if missing_repo_names: - msg = [] - msg.append("WARNING: One or more repositories " + \ - "have missing repo_name entries:") - msg.append("") - for p in missing_repo_names: - msg.append("\t%s/profiles/repo_name" % (p,)) - msg.append("") - msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \ - "should be a plain text file containing a unique " + \ - "name for the repository on the first line.", 70)) - msg.append("\n") - writemsg_level("".join("%s\n" % l for l in msg), - level=logging.WARNING, noiselevel=-1) - - return bool(missing_repo_names) - -def repo_name_duplicate_check(trees): - ignored_repos = {} - for root, root_trees in trees.items(): - if 'porttree' in root_trees: - portdb = root_trees['porttree'].dbapi - if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0': - for repo_name, paths in portdb.getIgnoredRepos(): - k = (root, repo_name, portdb.getRepositoryPath(repo_name)) - ignored_repos.setdefault(k, []).extend(paths) - - if ignored_repos: - msg = [] - msg.append('WARNING: One or more repositories ' + \ - 'have been ignored due to duplicate') - msg.append(' profiles/repo_name entries:') - msg.append('') - for k in sorted(ignored_repos): - msg.append(' %s overrides' % ", ".join(k)) - for path in ignored_repos[k]: - msg.append(' %s' % (path,)) - msg.append('') - msg.extend(' ' + x for x in textwrap.wrap( - "All profiles/repo_name entries must be unique in order " + \ - "to avoid having duplicates ignored. " + \ - "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \ - "/etc/make.conf if you would like to disable this warning.")) - msg.append("\n") - writemsg_level(''.join('%s\n' % l for l in msg), - level=logging.WARNING, noiselevel=-1) - - return bool(ignored_repos) - -def config_protect_check(trees): - for root, root_trees in trees.items(): - settings = root_trees["root_config"].settings - if not settings.get("CONFIG_PROTECT"): - msg = "!!! CONFIG_PROTECT is empty" - if settings["ROOT"] != "/": - msg += " for '%s'" % root - msg += "\n" - writemsg_level(msg, level=logging.WARN, noiselevel=-1) - def profile_check(trees, myaction): if myaction in ("help", "info", "search", "sync", "version"): return os.EX_OK @@ -1584,16 +968,6 @@ def profile_check(trees, myaction): return 1 return os.EX_OK -def check_procfs(): - procfs_path = '/proc' - if platform.system() not in ("Linux",) or \ - os.path.ismount(procfs_path): - return os.EX_OK - msg = "It seems that %s is not mounted. You have been warned." % procfs_path - writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)), - level=logging.ERROR, noiselevel=-1) - return 1 - def emerge_main(args=None): """ @param args: command arguments (default: sys.argv[1:]) @@ -1602,11 +976,12 @@ def emerge_main(args=None): if args is None: args = sys.argv[1:] - portage._disable_legacy_globals() - portage.dep._internal_warnings = True + args = portage._decode_argv(args) + # Disable color until we're sure that it should be enabled (after # EMERGE_DEFAULT_OPTS has been parsed). portage.output.havecolor = 0 + # This first pass is just for options that need to be known as early as # possible, such as --config-root. They will be parsed again later, # together with EMERGE_DEFAULT_OPTS (which may vary depending on the @@ -1618,428 +993,45 @@ def emerge_main(args=None): os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"] if "--root" in myopts: os.environ["ROOT"] = myopts["--root"] + if "--prefix" in myopts: + os.environ["EPREFIX"] = myopts["--prefix"] if "--accept-properties" in myopts: os.environ["ACCEPT_PROPERTIES"] = myopts["--accept-properties"] + if "--accept-restrict" in myopts: + os.environ["ACCEPT_RESTRICT"] = myopts["--accept-restrict"] + + # optimize --help (no need to load config / EMERGE_DEFAULT_OPTS) + if myaction == "help": + emerge_help() + return os.EX_OK + elif myaction == "moo": + print(COWSAY_MOO % platform.system()) + return os.EX_OK # Portage needs to ensure a sane umask for the files it creates. os.umask(0o22) - settings, trees, mtimedb = load_emerge_config() - portdb = trees[settings['EROOT']]['porttree'].dbapi - rval = profile_check(trees, myaction) + if myaction == "sync": + portage._sync_disabled_warnings = True + emerge_config = load_emerge_config( + action=myaction, args=myfiles, opts=myopts) + rval = profile_check(emerge_config.trees, emerge_config.action) if rval != os.EX_OK: return rval tmpcmdline = [] if "--ignore-default-opts" not in myopts: - tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split()) + tmpcmdline.extend(portage.util.shlex_split( + emerge_config.target_config.settings.get( + "EMERGE_DEFAULT_OPTS", ""))) tmpcmdline.extend(args) - myaction, myopts, myfiles = parse_opts(tmpcmdline) - - # skip global updates prior to sync, since it's called after sync - if myaction not in ('help', 'info', 'sync', 'version') and \ - myopts.get('--package-moves') != 'n' and \ - _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)): - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - - xterm_titles = "notitles" not in settings.features - if xterm_titles: - xtermTitle("emerge") - - if "--digest" in myopts: - os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest" - # Reload the whole config from scratch so that the portdbapi internal - # config is updated with new FEATURES. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - - # NOTE: adjust_configs() can map options to FEATURES, so any relevant - # options adjustments should be made prior to calling adjust_configs(). - if "--buildpkgonly" in myopts: - myopts["--buildpkg"] = True - - adjust_configs(myopts, trees) - apply_priorities(settings) - - if myaction == 'version': - writemsg_stdout(getportageversion( - settings["PORTDIR"], None, - settings.profile_path, settings["CHOST"], - trees[settings['EROOT']]['vartree'].dbapi) + '\n', noiselevel=-1) - return 0 - elif myaction == 'help': - _emerge.help.help() - return 0 - - spinner = stdout_spinner() - if "candy" in settings.features: - spinner.update = spinner.update_scroll - - if "--quiet" not in myopts: - portage.deprecated_profile_check(settings=settings) - if portage.const._ENABLE_REPO_NAME_WARN: - # Bug #248603 - Disable warnings about missing - # repo_name entries for stable branch. - repo_name_check(trees) - repo_name_duplicate_check(trees) - config_protect_check(trees) - check_procfs() - - if "getbinpkg" in settings.features: - myopts["--getbinpkg"] = True - - if "--getbinpkgonly" in myopts: - myopts["--getbinpkg"] = True - - if "--getbinpkgonly" in myopts: - myopts["--usepkgonly"] = True - - if "--getbinpkg" in myopts: - myopts["--usepkg"] = True - - if "--usepkgonly" in myopts: - myopts["--usepkg"] = True - - if "--buildpkgonly" in myopts: - # --buildpkgonly will not merge anything, so - # it cancels all binary package options. - for opt in ("--getbinpkg", "--getbinpkgonly", - "--usepkg", "--usepkgonly"): - myopts.pop(opt, None) - - for mytrees in trees.values(): - mydb = mytrees["porttree"].dbapi - # Freeze the portdbapi for performance (memoize all xmatch results). - mydb.freeze() - - if myaction in ('search', None) and \ - "--usepkg" in myopts: - # Populate the bintree with current --getbinpkg setting. - # This needs to happen before expand_set_arguments(), in case - # any sets use the bintree. - mytrees["bintree"].populate( - getbinpkgs="--getbinpkg" in myopts) - - del mytrees, mydb - - if "moo" in myfiles: - print(COWSAY_MOO % platform.system()) - msg = ("The above `emerge moo` display is deprecated. " - "Please use `emerge --moo` instead.") - for line in textwrap.wrap(msg, 50): - print(" %s %s" % (colorize("WARN", "*"), line)) - - for x in myfiles: - ext = os.path.splitext(x)[1] - if (ext == ".ebuild" or ext == ".tbz2") and os.path.exists(os.path.abspath(x)): - print(colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n")) - break - - root_config = trees[settings['EROOT']]['root_config'] - if myaction == "moo": - print(COWSAY_MOO % platform.system()) - return os.EX_OK - elif myaction == "list-sets": - writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets))) - return os.EX_OK - elif myaction == "check-news": - news_counts = count_unread_news( - root_config.trees["porttree"].dbapi, - root_config.trees["vartree"].dbapi) - if any(news_counts.values()): - display_news_notifications(news_counts) - elif "--quiet" not in myopts: - print("", colorize("GOOD", "*"), "No news items were found.") - return os.EX_OK - - ensure_required_sets(trees) - - # only expand sets for actions taking package arguments - oldargs = myfiles[:] - if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None): - myfiles, retval = expand_set_arguments(myfiles, myaction, root_config) - if retval != os.EX_OK: - return retval - - # Need to handle empty sets specially, otherwise emerge will react - # with the help message for empty argument lists - if oldargs and not myfiles: - print("emerge: no targets left after set expansion") - return 0 - - if ("--tree" in myopts) and ("--columns" in myopts): - print("emerge: can't specify both of \"--tree\" and \"--columns\".") - return 1 - - if '--emptytree' in myopts and '--noreplace' in myopts: - writemsg_level("emerge: can't specify both of " + \ - "\"--emptytree\" and \"--noreplace\".\n", - level=logging.ERROR, noiselevel=-1) - return 1 + emerge_config.action, emerge_config.opts, emerge_config.args = \ + parse_opts(tmpcmdline) - if ("--quiet" in myopts): - spinner.update = spinner.update_quiet - portage.util.noiselimit = -1 - - if "--fetch-all-uri" in myopts: - myopts["--fetchonly"] = True - - if "--skipfirst" in myopts and "--resume" not in myopts: - myopts["--resume"] = True - - # Allow -p to remove --ask - if "--pretend" in myopts: - myopts.pop("--ask", None) - - # forbid --ask when not in a terminal - # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway. - if ("--ask" in myopts) and (not sys.stdin.isatty()): - portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n", - noiselevel=-1) - return 1 - - if settings.get("PORTAGE_DEBUG", "") == "1": - spinner.update = spinner.update_quiet - portage.util.noiselimit = 0 - if "python-trace" in settings.features: - import portage.debug as portage_debug - portage_debug.set_trace(True) - - if not ("--quiet" in myopts): - if '--nospinner' in myopts or \ - settings.get('TERM') == 'dumb' or \ - not sys.stdout.isatty(): - spinner.update = spinner.update_basic - - if "--debug" in myopts: - print("myaction", myaction) - print("myopts", myopts) - - if not myaction and not myfiles and "--resume" not in myopts: - _emerge.help.help() - return 1 - - pretend = "--pretend" in myopts - fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts - buildpkgonly = "--buildpkgonly" in myopts - - # check if root user is the current user for the actions where emerge needs this - if portage.secpass < 2: - # We've already allowed "--version" and "--help" above. - if "--pretend" not in myopts and myaction not in ("search","info"): - need_superuser = myaction in ('clean', 'depclean', 'deselect', - 'prune', 'unmerge') or not \ - (fetchonly or \ - (buildpkgonly and secpass >= 1) or \ - myaction in ("metadata", "regen", "sync")) - if portage.secpass < 1 or \ - need_superuser: - if need_superuser: - access_desc = "superuser" - else: - access_desc = "portage group" - # Always show portage_group_warning() when only portage group - # access is required but the user is not in the portage group. - from portage.data import portage_group_warning - if "--ask" in myopts: - writemsg_stdout("This action requires %s access...\n" % \ - (access_desc,), noiselevel=-1) - if portage.secpass < 1 and not need_superuser: - portage_group_warning() - if userquery("Would you like to add --pretend to options?", - "--ask-enter-invalid" in myopts) == "No": - return 128 + signal.SIGINT - myopts["--pretend"] = True - del myopts["--ask"] - else: - sys.stderr.write(("emerge: %s access is required\n") \ - % access_desc) - if portage.secpass < 1 and not need_superuser: - portage_group_warning() - return 1 - - # Disable emergelog for everything except build or unmerge operations. - # This helps minimize parallel emerge.log entries that can confuse log - # parsers like genlop. - disable_emergelog = False - for x in ("--pretend", "--fetchonly", "--fetch-all-uri"): - if x in myopts: - disable_emergelog = True - break - if disable_emergelog: - pass - elif myaction in ("search", "info"): - disable_emergelog = True - elif portage.data.secpass < 1: - disable_emergelog = True - - _emerge.emergelog._disable = disable_emergelog - - if not disable_emergelog: - if 'EMERGE_LOG_DIR' in settings: - try: - # At least the parent needs to exist for the lock file. - portage.util.ensure_dirs(settings['EMERGE_LOG_DIR']) - except portage.exception.PortageException as e: - writemsg_level("!!! Error creating directory for " + \ - "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \ - (settings['EMERGE_LOG_DIR'], e), - noiselevel=-1, level=logging.ERROR) - portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) - else: - _emerge.emergelog._emerge_log_dir = settings["EMERGE_LOG_DIR"] - else: - _emerge.emergelog._emerge_log_dir = os.path.join(os.sep, - settings["EPREFIX"].lstrip(os.sep), "var", "log") - portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) - - if not "--pretend" in myopts: - emergelog(xterm_titles, "Started emerge on: "+\ - _unicode_decode( - time.strftime("%b %d, %Y %H:%M:%S", time.localtime()), - encoding=_encodings['content'], errors='replace')) - myelogstr="" - if myopts: - opt_list = [] - for opt, arg in myopts.items(): - if arg is True: - opt_list.append(opt) - elif isinstance(arg, list): - # arguments like --exclude that use 'append' action - for x in arg: - opt_list.append("%s=%s" % (opt, x)) - else: - opt_list.append("%s=%s" % (opt, arg)) - myelogstr=" ".join(opt_list) - if myaction: - myelogstr += " --" + myaction - if myfiles: - myelogstr += " " + " ".join(oldargs) - emergelog(xterm_titles, " *** emerge " + myelogstr) - del oldargs - - def emergeexitsig(signum, frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % {"signal":signum}) - sys.exit(128 + signum) - signal.signal(signal.SIGINT, emergeexitsig) - signal.signal(signal.SIGTERM, emergeexitsig) - - def emergeexit(): - """This gets out final log message in before we quit.""" - if "--pretend" not in myopts: - emergelog(xterm_titles, " *** terminating.") - if xterm_titles: - xtermTitleReset() - portage.atexit_register(emergeexit) - - if myaction in ("config", "metadata", "regen", "sync"): - if "--pretend" in myopts: - sys.stderr.write(("emerge: The '%s' action does " + \ - "not support '--pretend'.\n") % myaction) - return 1 - - if "sync" == myaction: - return action_sync(settings, trees, mtimedb, myopts, myaction) - elif "metadata" == myaction: - action_metadata(settings, portdb, myopts) - elif myaction=="regen": - validate_ebuild_environment(trees) - return action_regen(settings, portdb, myopts.get("--jobs"), - myopts.get("--load-average")) - # HELP action - elif "config"==myaction: - validate_ebuild_environment(trees) - action_config(settings, trees, myopts, myfiles) - - # SEARCH action - elif "search"==myaction: - validate_ebuild_environment(trees) - action_search(trees[settings['EROOT']]['root_config'], - myopts, myfiles, spinner) - - elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'): - validate_ebuild_environment(trees) - rval = action_uninstall(settings, trees, mtimedb["ldpath"], - myopts, myaction, myfiles, spinner) - if not (myaction == 'deselect' or buildpkgonly or fetchonly or pretend): - post_emerge(myaction, myopts, myfiles, settings['EROOT'], - trees, mtimedb, rval) - return rval - - elif myaction == 'info': - - # Ensure atoms are valid before calling unmerge(). - vardb = trees[settings['EROOT']]['vartree'].dbapi - portdb = trees[settings['EROOT']]['porttree'].dbapi - bindb = trees[settings['EROOT']]["bintree"].dbapi - valid_atoms = [] - for x in myfiles: - if is_valid_package_atom(x, allow_repo=True): - try: - #look at the installed files first, if there is no match - #look at the ebuilds, since EAPI 4 allows running pkg_info - #on non-installed packages - valid_atom = dep_expand(x, mydb=vardb, settings=settings) - if valid_atom.cp.split("/")[0] == "null": - valid_atom = dep_expand(x, mydb=portdb, settings=settings) - if valid_atom.cp.split("/")[0] == "null" and "--usepkg" in myopts: - valid_atom = dep_expand(x, mydb=bindb, settings=settings) - valid_atoms.append(valid_atom) - except portage.exception.AmbiguousPackageName as e: - msg = "The short ebuild name \"" + x + \ - "\" is ambiguous. Please specify " + \ - "one of the following " + \ - "fully-qualified ebuild names instead:" - for line in textwrap.wrap(msg, 70): - writemsg_level("!!! %s\n" % (line,), - level=logging.ERROR, noiselevel=-1) - for i in e.args[0]: - writemsg_level(" %s\n" % colorize("INFORM", i), - level=logging.ERROR, noiselevel=-1) - writemsg_level("\n", level=logging.ERROR, noiselevel=-1) - return 1 - continue - msg = [] - msg.append("'%s' is not a valid package atom." % (x,)) - msg.append("Please check ebuild(5) for full details.") - writemsg_level("".join("!!! %s\n" % line for line in msg), - level=logging.ERROR, noiselevel=-1) - return 1 - - return action_info(settings, trees, myopts, valid_atoms) - - # "update", "system", or just process files: - else: - validate_ebuild_environment(trees) - - for x in myfiles: - if x.startswith(SETPREFIX) or \ - is_valid_package_atom(x, allow_repo=True): - continue - if x[:1] == os.sep: - continue - try: - os.lstat(x) + try: + return run_action(emerge_config) + finally: + # Call destructors for our portdbapi instances. + for x in emerge_config.trees.values(): + if "porttree" in x.lazy_items: continue - except OSError: - pass - msg = [] - msg.append("'%s' is not a valid package atom." % (x,)) - msg.append("Please check ebuild(5) for full details.") - writemsg_level("".join("!!! %s\n" % line for line in msg), - level=logging.ERROR, noiselevel=-1) - return 1 - - # GLEP 42 says to display news *after* an emerge --pretend - if "--pretend" not in myopts: - display_news_notification(root_config, myopts) - retval = action_build(settings, trees, mtimedb, - myopts, myaction, myfiles, spinner) - post_emerge(myaction, myopts, myfiles, settings['EROOT'], - trees, mtimedb, retval) - - return retval + x["porttree"].dbapi.close_caches() diff --git a/portage_with_autodep/pym/_emerge/main.pyo b/portage_with_autodep/pym/_emerge/main.pyo Binary files differindex aaeb5b9..2047458 100644 --- a/portage_with_autodep/pym/_emerge/main.pyo +++ b/portage_with_autodep/pym/_emerge/main.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/__init__.pyo b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo Binary files differindex 5c1b374..a5d6b7f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.py b/portage_with_autodep/pym/_emerge/resolver/backtracking.py index f2857b0..c29b9d4 100644 --- a/portage_with_autodep/pym/_emerge/resolver/backtracking.py +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import copy @@ -7,7 +7,8 @@ class BacktrackParameter(object): __slots__ = ( "needed_unstable_keywords", "runtime_pkg_mask", "needed_use_config_changes", "needed_license_changes", - "rebuild_list", "reinstall_list", "needed_p_mask_changes" + "prune_rebuilds", "rebuild_list", "reinstall_list", "needed_p_mask_changes", + "slot_operator_mask_built", "slot_operator_replace_installed" ) def __init__(self): @@ -18,6 +19,9 @@ class BacktrackParameter(object): self.needed_license_changes = {} self.rebuild_list = set() self.reinstall_list = set() + self.slot_operator_replace_installed = set() + self.slot_operator_mask_built = set() + self.prune_rebuilds = False def __deepcopy__(self, memo=None): if memo is None: @@ -29,11 +33,18 @@ class BacktrackParameter(object): #to our sets and dicts. The existing content is immutable. result.needed_unstable_keywords = copy.copy(self.needed_unstable_keywords) result.needed_p_mask_changes = copy.copy(self.needed_p_mask_changes) - result.runtime_pkg_mask = copy.copy(self.runtime_pkg_mask) result.needed_use_config_changes = copy.copy(self.needed_use_config_changes) result.needed_license_changes = copy.copy(self.needed_license_changes) result.rebuild_list = copy.copy(self.rebuild_list) result.reinstall_list = copy.copy(self.reinstall_list) + result.slot_operator_replace_installed = copy.copy(self.slot_operator_replace_installed) + result.slot_operator_mask_built = self.slot_operator_mask_built.copy() + result.prune_rebuilds = self.prune_rebuilds + + # runtime_pkg_mask contains nested dicts that must also be copied + result.runtime_pkg_mask = {} + for k, v in self.runtime_pkg_mask.items(): + result.runtime_pkg_mask[k] = copy.copy(v) return result @@ -44,7 +55,10 @@ class BacktrackParameter(object): self.needed_use_config_changes == other.needed_use_config_changes and \ self.needed_license_changes == other.needed_license_changes and \ self.rebuild_list == other.rebuild_list and \ - self.reinstall_list == other.reinstall_list + self.reinstall_list == other.reinstall_list and \ + self.slot_operator_replace_installed == other.slot_operator_replace_installed and \ + self.slot_operator_mask_built == other.slot_operator_mask_built and \ + self.prune_rebuilds == other.prune_rebuilds class _BacktrackNode(object): @@ -114,9 +128,10 @@ class Backtracker(object): before, we revert the mask for other packages (bug 375573). """ - for pkg in runtime_pkg_mask: + for pkg, mask_info in runtime_pkg_mask.items(): - if "missing dependency" in runtime_pkg_mask[pkg]: + if "missing dependency" in mask_info or \ + "slot_operator_mask_built" in mask_info: continue entry_is_valid = False @@ -131,6 +146,13 @@ class Backtracker(object): return True + def _feedback_slot_conflicts(self, conflicts_data): + # Only create BacktrackNode instances for the first + # conflict which occurred, since the conflicts that + # occurred later may have been caused by the first + # conflict. + self._feedback_slot_conflict(conflicts_data[0]) + def _feedback_slot_conflict(self, conflict_data): for pkg, parent_atoms in conflict_data: new_node = copy.deepcopy(self._current_node) @@ -174,10 +196,30 @@ class Backtracker(object): elif change == "needed_use_config_changes": for pkg, (new_use, new_changes) in data: para.needed_use_config_changes[pkg] = (new_use, new_changes) + elif change == "slot_conflict_abi": + new_node.terminal = False + elif change == "slot_operator_mask_built": + para.slot_operator_mask_built.update(data) + for pkg, mask_reasons in data.items(): + para.runtime_pkg_mask.setdefault(pkg, + {}).update(mask_reasons) + elif change == "slot_operator_replace_installed": + para.slot_operator_replace_installed.update(data) elif change == "rebuild_list": para.rebuild_list.update(data) elif change == "reinstall_list": para.reinstall_list.update(data) + elif change == "prune_rebuilds": + para.prune_rebuilds = True + para.slot_operator_replace_installed.clear() + for pkg in para.slot_operator_mask_built: + runtime_masks = para.runtime_pkg_mask.get(pkg) + if runtime_masks is None: + continue + runtime_masks.pop("slot_operator_mask_built", None) + if not runtime_masks: + para.runtime_pkg_mask.pop(pkg) + para.slot_operator_mask_built.clear() self._add(new_node, explore=explore) self._current_node = new_node @@ -196,7 +238,7 @@ class Backtracker(object): #There is at most one of the following types of conflicts for a given restart. if "slot conflict" in infos: - self._feedback_slot_conflict(infos["slot conflict"]) + self._feedback_slot_conflicts(infos["slot conflict"]) elif "missing dependency" in infos: self._feedback_missing_dep(infos["missing dependency"]) diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo Binary files differindex d989c15..bc8338e 100644 --- a/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py index aca81fa..b710671 100644 --- a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py @@ -1,7 +1,7 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals from itertools import chain, product import logging @@ -11,6 +11,7 @@ from portage.exception import InvalidDependString from portage.output import colorize from portage.util import writemsg_level from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange +from _emerge.Package import Package class circular_dependency_handler(object): @@ -61,8 +62,7 @@ class circular_dependency_handler(object): node = nodes[0] display_order.append(node) tempgraph.remove(node) - display_order.reverse() - return display_order + return tuple(display_order) def _prepare_circular_dep_message(self): """ @@ -113,9 +113,10 @@ class circular_dependency_handler(object): parent_atoms = self.all_parent_atoms.get(pkg) if priorities[-1].buildtime: - dep = parent.metadata["DEPEND"] + dep = " ".join(parent._metadata[k] + for k in Package._buildtime_keys) elif priorities[-1].runtime: - dep = parent.metadata["RDEPEND"] + dep = parent._metadata["RDEPEND"] for ppkg, atom in parent_atoms: if ppkg == parent: @@ -125,7 +126,7 @@ class circular_dependency_handler(object): try: affecting_use = extract_affecting_use(dep, parent_atom, - eapi=parent.metadata["EAPI"]) + eapi=parent.eapi) except InvalidDependString: if not parent.installed: raise @@ -144,7 +145,8 @@ class circular_dependency_handler(object): #If any of the flags we're going to touch is in REQUIRED_USE, add all #other flags in REQUIRED_USE to affecting_use, to not lose any solution. required_use_flags = get_required_use_flags( - parent.metadata.get("REQUIRED_USE", "")) + parent._metadata.get("REQUIRED_USE", ""), + eapi=parent.eapi) if affecting_use.intersection(required_use_flags): # TODO: Find out exactly which REQUIRED_USE flags are @@ -186,9 +188,11 @@ class circular_dependency_handler(object): parent_atom not in reduced_dep: #We found an assignment that removes the atom from 'dep'. #Make sure it doesn't conflict with REQUIRED_USE. - required_use = parent.metadata.get("REQUIRED_USE", "") + required_use = parent._metadata.get("REQUIRED_USE", "") - if check_required_use(required_use, current_use, parent.iuse.is_valid_flag): + if check_required_use(required_use, current_use, + parent.iuse.is_valid_flag, + eapi=parent.eapi): use = self.depgraph._pkg_use_enabled(parent) solution = set() for flag, state in zip(affecting_use, use_state): diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo Binary files differindex c1f95dc..17fb71d 100644 --- a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output.py b/portage_with_autodep/pym/_emerge/resolver/output.py index 1208bf9..3e8552f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output.py +++ b/portage_with_autodep/pym/_emerge/resolver/output.py @@ -1,26 +1,30 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Resolver output display operation. """ +from __future__ import unicode_literals + __all__ = ( "Display", ) import sys +import portage from portage import os -from portage import _unicode_decode from portage.dbapi.dep_expand import dep_expand -from portage.dep import cpvequal, _repo_separator +from portage.dep import cpvequal, _repo_separator, _slot_separator +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidDependString, SignatureException +from portage.package.ebuild.config import _get_feature_flags from portage.package.ebuild._spawn_nofetch import spawn_nofetch from portage.output import ( blue, colorize, create_color_func, - darkblue, darkgreen, green, nc_len, red, teal, turquoise, yellow ) + darkblue, darkgreen, green, nc_len, teal) bad = create_color_func("BAD") from portage.util import writemsg_stdout -from portage.versions import best, catpkgsplit +from portage.versions import best from _emerge.Blocker import Blocker from _emerge.create_world_atom import create_world_atom @@ -30,7 +34,9 @@ from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice if sys.hexversion >= 0x3000000: basestring = str - + _unicode = str +else: + _unicode = unicode class Display(object): """Formats and outputs the depgrah supplied it for merge/re-merge, etc. @@ -54,11 +60,6 @@ class Display(object): self.oldlp = None self.myfetchlist = None self.indent = '' - self.is_new = True - self.cur_use = None - self.cur_iuse = None - self.old_use = '' - self.old_iuse = '' self.use_expand = None self.use_expand_hidden = None self.pkgsettings = None @@ -68,93 +69,54 @@ class Display(object): self.blocker_style = None - def _blockers(self, pkg, fetch_symbol): - """Processes pkg for blockers and adds colorized strings to + def _blockers(self, blocker): + """Adds colorized strings to self.print_msg and self.blockers - @param pkg: _emerge.Package.Package instance - @param fetch_symbol: string + @param blocker: _emerge.Blocker.Blocker instance @rtype: bool Modifies class globals: self.blocker_style, self.resolved, self.print_msg """ - if pkg.satisfied: + if blocker.satisfied: self.blocker_style = "PKG_BLOCKER_SATISFIED" - addl = "%s %s " % (colorize(self.blocker_style, "b"), - fetch_symbol) + addl = "%s " % (colorize(self.blocker_style, "b"),) else: self.blocker_style = "PKG_BLOCKER" - addl = "%s %s " % (colorize(self.blocker_style, "B"), - fetch_symbol) + addl = "%s " % (colorize(self.blocker_style, "B"),) addl += self.empty_space_in_brackets() self.resolved = dep_expand( - str(pkg.atom).lstrip("!"), mydb=self.vardb, + _unicode(blocker.atom).lstrip("!"), mydb=self.vardb, settings=self.pkgsettings ) if self.conf.columns and self.conf.quiet: - addl += " " + colorize(self.blocker_style, str(self.resolved)) + addl += " " + colorize(self.blocker_style, _unicode(self.resolved)) else: addl = "[%s %s] %s%s" % \ (colorize(self.blocker_style, "blocks"), addl, self.indent, - colorize(self.blocker_style, str(self.resolved)) + colorize(self.blocker_style, _unicode(self.resolved)) ) - block_parents = self.conf.blocker_parents.parent_nodes(pkg) - block_parents = set([pnode[2] for pnode in block_parents]) + block_parents = self.conf.blocker_parents.parent_nodes(blocker) + block_parents = set(_unicode(pnode.cpv) for pnode in block_parents) block_parents = ", ".join(block_parents) - if self.resolved != pkg[2]: + if blocker.atom.blocker.overlap.forbid: + blocking_desc = "hard blocking" + else: + blocking_desc = "blocking" + if self.resolved != blocker.atom: addl += colorize(self.blocker_style, - " (\"%s\" is blocking %s)") % \ - (str(pkg.atom).lstrip("!"), block_parents) + " (\"%s\" is %s %s)" % + (_unicode(blocker.atom).lstrip("!"), + blocking_desc, block_parents)) else: addl += colorize(self.blocker_style, - " (is blocking %s)") % block_parents - if isinstance(pkg, Blocker) and pkg.satisfied: - if self.conf.columns: - return True - self.print_msg.append(addl) + " (is %s %s)" % (blocking_desc, block_parents)) + if blocker.satisfied: + if not self.conf.columns: + self.print_msg.append(addl) else: self.blockers.append(addl) - return False - - - def _display_use(self, pkg, myoldbest, myinslotlist): - """ USE flag display - - @param pkg: _emerge.Package.Package instance - @param myoldbest: list of installed versions - @param myinslotlist: list of installed slots - Modifies class globals: self.forced_flags, self.cur_iuse, - self.old_iuse, self.old_use, self.use_expand - """ - - self.forced_flags = set() - self.forced_flags.update(pkg.use.force) - self.forced_flags.update(pkg.use.mask) - - self.cur_use = [flag for flag in self.conf.pkg_use_enabled(pkg) \ - if flag in pkg.iuse.all] - self.cur_iuse = sorted(pkg.iuse.all) - - if myoldbest and myinslotlist: - previous_cpv = myoldbest[0].cpv - else: - previous_cpv = pkg.cpv - if self.vardb.cpv_exists(previous_cpv): - previous_pkg = self.vardb.match_pkgs('=' + previous_cpv)[0] - self.old_iuse = sorted(previous_pkg.iuse.all) - self.old_use = previous_pkg.use.enabled - self.is_new = False - else: - self.old_iuse = [] - self.old_use = [] - self.is_new = True - - self.old_use = [flag for flag in self.old_use if flag in self.old_iuse] - - self.use_expand = pkg.use.expand - self.use_expand_hidden = pkg.use.expand_hidden - return def include_mask_str(self): return self.conf.verbosity > 1 @@ -219,13 +181,40 @@ class Display(object): return ret - def recheck_hidden(self, pkg): - """ Prevent USE_EXPAND_HIDDEN flags from being hidden if they - are the only thing that triggered reinstallation. + def _display_use(self, pkg, pkg_info): + """ USE flag display @param pkg: _emerge.Package.Package instance - Modifies self.use_expand_hidden, self.use_expand, self.verboseadd + @param pkg_info: PkgInfo instance + Modifies self.use_expand_hidden, self.use_expand, self.verboseadd, + self.forced_flags """ + + self.forced_flags = set() + self.forced_flags.update(pkg.use.force) + self.forced_flags.update(pkg.use.mask) + + cur_use = [flag for flag in self.conf.pkg_use_enabled(pkg) \ + if flag in pkg.iuse.all] + cur_iuse = sorted(pkg.iuse.all) + + if pkg_info.previous_pkg is not None: + previous_pkg = pkg_info.previous_pkg + old_iuse = sorted(previous_pkg.iuse.all) + old_use = previous_pkg.use.enabled + is_new = False + else: + old_iuse = [] + old_use = [] + is_new = True + + old_use = [flag for flag in old_use if flag in old_iuse] + + self.use_expand = pkg.use.expand + self.use_expand_hidden = pkg.use.expand_hidden + + # Prevent USE_EXPAND_HIDDEN flags from being hidden if they + # are the only thing that triggered reinstallation. reinst_flags_map = {} reinstall_for_flags = self.conf.reinstall_nodes.get(pkg) reinst_expand_map = None @@ -246,13 +235,14 @@ class Display(object): reinst_expand_map) cur_iuse_map, iuse_forced = \ - self.map_to_use_expand(self.cur_iuse, forced_flags=True) - cur_use_map = self.map_to_use_expand(self.cur_use) - old_iuse_map = self.map_to_use_expand(self.old_iuse) - old_use_map = self.map_to_use_expand(self.old_use) + self.map_to_use_expand(cur_iuse, forced_flags=True) + cur_use_map = self.map_to_use_expand(cur_use) + old_iuse_map = self.map_to_use_expand(old_iuse) + old_use_map = self.map_to_use_expand(old_use) use_expand = sorted(self.use_expand) use_expand.insert(0, "USE") + feature_flags = _get_feature_flags(_get_eapi_attrs(pkg.eapi)) for key in use_expand: if key in self.use_expand_hidden: @@ -260,7 +250,7 @@ class Display(object): self.verboseadd += _create_use_string(self.conf, key.upper(), cur_iuse_map[key], iuse_forced[key], cur_use_map[key], old_iuse_map[key], - old_use_map[key], self.is_new, + old_use_map[key], is_new, feature_flags, reinst_flags_map.get(key)) return @@ -318,13 +308,14 @@ class Display(object): kwargs["myrepo"] = pkg.repo myfilesdict = None try: - myfilesdict = db.getfetchsizes(pkg.cpv, **kwargs) + myfilesdict = db.getfetchsizes(pkg.cpv, + **portage._native_kwargs(kwargs)) except InvalidDependString as e: # FIXME: validate SRC_URI earlier depstr, = db.aux_get(pkg.cpv, ["SRC_URI"], myrepo=pkg.repo) show_invalid_depstring_notice( - pkg, depstr, str(e)) + pkg, depstr, _unicode(e)) raise except SignatureException: # missing/invalid binary package SIZE signature @@ -343,15 +334,13 @@ class Display(object): if self.quiet_repo_display: # overlay verbose # assign index for a previous version in the same slot - slot_matches = self.vardb.match(pkg.slot_atom) - if slot_matches: - repo_name_prev = self.vardb.aux_get(slot_matches[0], - ["repository"])[0] + if pkg_info.previous_pkg is not None: + repo_name_prev = pkg_info.previous_pkg.repo else: repo_name_prev = None # now use the data to generate output - if pkg.installed or not slot_matches: + if pkg.installed or pkg_info.previous_pkg is None: self.repoadd = self.conf.repo_display.repoStr( pkg_info.repo_path_real) else: @@ -370,58 +359,86 @@ class Display(object): repoadd_set.add(self.repoadd) - def convert_myoldbest(self, pkg, myoldbest): + def convert_myoldbest(self, pkg, pkg_info): """converts and colorizes a version list to a string @param pkg: _emerge.Package.Package instance - @param myoldbest: list + @param pkg_info: dictionary @rtype string. """ + myoldbest = pkg_info.oldbest_list # Convert myoldbest from a list to a string. myoldbest_str = "" if myoldbest: versions = [] for pos, old_pkg in enumerate(myoldbest): - key = catpkgsplit(old_pkg.cpv)[2] + "-" + catpkgsplit(old_pkg.cpv)[3] + key = old_pkg.version if key[-3:] == "-r0": key = key[:-3] - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])): - key += _repo_separator + old_pkg.repo + if self.conf.verbosity == 3: + if pkg_info.attr_display.new_slot: + key += _slot_separator + old_pkg.slot + if old_pkg.slot != old_pkg.sub_slot: + key += "/" + old_pkg.sub_slot + elif any(x.slot + "/" + x.sub_slot != "0/0" for x in myoldbest + [pkg]): + key += _slot_separator + old_pkg.slot + if old_pkg.slot != old_pkg.sub_slot or \ + old_pkg.slot == pkg.slot and old_pkg.sub_slot != pkg.sub_slot: + key += "/" + old_pkg.sub_slot + if not self.quiet_repo_display and (self.verbose_main_repo_display or + self.portdb.repositories.mainRepo() is None or + any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])): + key += _repo_separator + old_pkg.repo versions.append(key) myoldbest_str = blue("["+", ".join(versions)+"]") return myoldbest_str + def _append_slot(self, pkg_str, pkg, pkg_info): + """Potentially appends slot and subslot to package string. - def set_interactive(self, pkg, ordered, addl): - """Increments counters.interactive if the pkg is to - be merged and it's metadata has interactive set True + @param pkg_str: string + @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary + @rtype string + """ + if pkg_info.attr_display.new_slot: + pkg_str += _slot_separator + pkg_info.slot + if pkg_info.slot != pkg_info.sub_slot: + pkg_str += "/" + pkg_info.sub_slot + elif any(x.slot + "/" + x.sub_slot != "0/0" for x in pkg_info.oldbest_list + [pkg]): + pkg_str += _slot_separator + pkg_info.slot + if pkg_info.slot != pkg_info.sub_slot or \ + any(x.slot == pkg_info.slot and x.sub_slot != pkg_info.sub_slot for x in pkg_info.oldbest_list): + pkg_str += "/" + pkg_info.sub_slot + return pkg_str + + def _append_repository(self, pkg_str, pkg, pkg_info): + """Potentially appends repository to package string. + @param pkg_str: string @param pkg: _emerge.Package.Package instance - @param ordered: boolean - @param addl: already defined string to add to + @param pkg_info: dictionary + @rtype string """ - if 'interactive' in pkg.metadata.properties and \ - pkg.operation == 'merge': - addl = colorize("WARN", "I") + addl[1:] - if ordered: - self.counters.interactive += 1 - return addl - - def _set_non_root_columns(self, addl, pkg_info, pkg): + if not self.quiet_repo_display and (self.verbose_main_repo_display or + self.portdb.repositories.mainRepo() is None or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + pkg_str += _repo_separator + pkg.repo + return pkg_str + + def _set_non_root_columns(self, pkg, pkg_info): """sets the indent level and formats the output - @param addl: already defined string to add to - @param pkg_info: dictionary @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary @rtype string """ ver_str = pkg_info.ver - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - ver_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + ver_str = self._append_slot(ver_str, pkg, pkg_info) + ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = addl + " " + self.indent + \ + myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+darkblue(" "+ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -434,7 +451,8 @@ class Display(object): self.indent, self.pkgprint(pkg.cp, pkg_info)) else: myprint = "[%s %s] %s%s" % \ - (self.pkgprint(pkg.type_name, pkg_info), addl, + (self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display, self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) @@ -446,21 +464,20 @@ class Display(object): return myprint - def _set_root_columns(self, addl, pkg_info, pkg): + def _set_root_columns(self, pkg, pkg_info): """sets the indent level and formats the output - @param addl: already defined string to add to - @param pkg_info: dictionary @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary @rtype string Modifies self.verboseadd """ ver_str = pkg_info.ver - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - ver_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + ver_str = self._append_slot(ver_str, pkg, pkg_info) + ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = addl + " " + self.indent + \ + myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+" "+green(ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -473,7 +490,8 @@ class Display(object): addl, self.indent, self.pkgprint(pkg.cp, pkg_info)) else: myprint = "[%s %s] %s%s" % \ - (self.pkgprint(pkg.type_name, pkg_info), addl, + (self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display, self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) @@ -484,18 +502,17 @@ class Display(object): return myprint - def _set_no_columns(self, pkg, pkg_info, addl): + def _set_no_columns(self, pkg, pkg_info): """prints pkg info without column indentation. @param pkg: _emerge.Package.Package instance @param pkg_info: dictionary - @param addl: the current text to add for the next line to output @rtype the updated addl """ pkg_str = pkg.cpv - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - pkg_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + pkg_str = self._append_slot(pkg_str, pkg, pkg_info) + pkg_str = self._append_repository(pkg_str, pkg, pkg_info) if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] %s%s %s" % \ @@ -506,53 +523,10 @@ class Display(object): else: myprint = "[%s %s] %s%s %s" % \ (self.pkgprint(pkg.type_name, pkg_info), - addl, self.indent, + pkg_info.attr_display, self.indent, self.pkgprint(pkg_str, pkg_info), pkg_info.oldbest) return myprint - - def _insert_slot(self, pkg, pkg_info, myinslotlist): - """Adds slot info to the message - - @return addl: formatted slot info - @return myoldbest: installed version list - Modifies self.counters.downgrades, self.counters.upgrades, - self.counters.binary - """ - addl = " " + pkg_info.fetch_symbol - if not cpvequal(pkg.cpv, - best([pkg.cpv] + [x.cpv for x in myinslotlist])): - # Downgrade in slot - addl += turquoise("U")+blue("D") - if pkg_info.ordered: - self.counters.downgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - else: - # Update in slot - addl += turquoise("U") + " " - if pkg_info.ordered: - self.counters.upgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl - - - def _new_slot(self, pkg, pkg_info): - """New slot, mark it new. - - @return addl: formatted slot info - @return myoldbest: installed version list - Modifies self.counters.newslot, self.counters.binary - """ - addl = " " + green("NS") + pkg_info.fetch_symbol + " " - if pkg_info.ordered: - self.counters.newslot += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl - - def print_messages(self, show_repos): """Performs the actual output printing of the pre-formatted messages @@ -588,9 +562,9 @@ class Display(object): """ writemsg_stdout('\n%s\n' % (self.counters,), noiselevel=-1) if show_repos: - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that RepoDisplay.__unicode__() is called in python2. - writemsg_stdout(_unicode_decode("%s") % (self.conf.repo_display,), + writemsg_stdout("%s" % (self.conf.repo_display,), noiselevel=-1) return @@ -642,15 +616,24 @@ class Display(object): self.counters.restrict_fetch_satisfied """ pkg_info = PkgInfo() + pkg_info.cp = pkg.cp + pkg_info.ver = self.get_ver_str(pkg) + pkg_info.slot = pkg.slot + pkg_info.sub_slot = pkg.sub_slot + pkg_info.repo_name = pkg.repo pkg_info.ordered = ordered - pkg_info.fetch_symbol = " " pkg_info.operation = pkg.operation pkg_info.merge = ordered and pkg_info.operation == "merge" if not pkg_info.merge and pkg_info.operation == "merge": pkg_info.operation = "nomerge" pkg_info.built = pkg.type_name != "ebuild" pkg_info.ebuild_path = None - pkg_info.repo_name = pkg.repo + if ordered: + if pkg_info.merge: + if pkg.type_name == "binary": + self.counters.binary += 1 + elif pkg_info.operation == "uninstall": + self.counters.uninst += 1 if pkg.type_name == "ebuild": pkg_info.ebuild_path = self.portdb.findname( pkg.cpv, myrepo=pkg_info.repo_name) @@ -660,22 +643,30 @@ class Display(object): pkg_info.repo_path_real = os.path.dirname(os.path.dirname( os.path.dirname(pkg_info.ebuild_path))) else: - pkg_info.repo_path_real = \ - self.portdb.getRepositoryPath(pkg.metadata["repository"]) + pkg_info.repo_path_real = self.portdb.getRepositoryPath(pkg.repo) pkg_info.use = list(self.conf.pkg_use_enabled(pkg)) if not pkg.built and pkg.operation == 'merge' and \ - 'fetch' in pkg.metadata.restrict: + 'fetch' in pkg.restrict: if pkg_info.ordered: self.counters.restrict_fetch += 1 + pkg_info.attr_display.fetch_restrict = True if not self.portdb.getfetchsizes(pkg.cpv, useflags=pkg_info.use, myrepo=pkg.repo): - pkg_info.fetch_symbol = green("f") + pkg_info.attr_display.fetch_restrict_satisfied = True if pkg_info.ordered: self.counters.restrict_fetch_satisfied += 1 else: - pkg_info.fetch_symbol = red("F") if pkg_info.ebuild_path is not None: self.restrict_fetch_list[pkg] = pkg_info + + if self.vardb.cpv_exists(pkg.cpv): + # Do a cpv match first, in case the SLOT has changed. + pkg_info.previous_pkg = self.vardb.match_pkgs('=' + pkg.cpv)[0] + else: + slot_matches = self.vardb.match_pkgs(pkg.slot_atom) + if slot_matches: + pkg_info.previous_pkg = slot_matches[0] + return pkg_info @@ -686,15 +677,14 @@ class Display(object): @param pkg_info: dictionay Modifies self.changelogs """ - inst_matches = self.vardb.match(pkg.slot_atom) - if inst_matches: + if pkg_info.previous_pkg is not None: ebuild_path_cl = pkg_info.ebuild_path if ebuild_path_cl is None: # binary package ebuild_path_cl = self.portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path_cl is not None: self.changelogs.extend(_calc_changelog( - ebuild_path_cl, inst_matches[0], pkg.cpv)) + ebuild_path_cl, pkg_info.previous_pkg, pkg.cpv)) return @@ -734,12 +724,10 @@ class Display(object): @param pkg: _emerge.Package.Package instance @rtype string """ - ver_str = list(catpkgsplit(pkg.cpv)[2:]) - if ver_str[1] == "r0": - ver_str[1] = "" - else: - ver_str[1] = "-" + ver_str[1] - return ver_str[0]+ver_str[1] + ver_str = pkg.cpv.version + if ver_str.endswith("-r0"): + ver_str = ver_str[:-3] + return ver_str def _get_installed_best(self, pkg, pkg_info): @@ -751,24 +739,21 @@ class Display(object): @param pkg: _emerge.Package.Package instance @param pkg_info: dictionay @rtype addl, myoldbest: list, myinslotlist: list - Modifies self.counters.reinst, self.counters.binary, self.counters.new + Modifies self.counters.reinst, self.counters.new """ myoldbest = [] myinslotlist = None installed_versions = self.vardb.match_pkgs(pkg.cp) if self.vardb.cpv_exists(pkg.cpv): - addl = " "+yellow("R")+pkg_info.fetch_symbol+" " - installed_version = self.vardb.match_pkgs(pkg.cpv)[0] - if not self.quiet_repo_display and installed_version.repo != pkg.repo: + pkg_info.attr_display.replace = True + installed_version = pkg_info.previous_pkg + if installed_version.slot != pkg.slot or installed_version.sub_slot != pkg.sub_slot or \ + not self.quiet_repo_display and installed_version.repo != pkg.repo: myoldbest = [installed_version] if pkg_info.ordered: if pkg_info.merge: self.counters.reinst += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - elif pkg_info.operation == "uninstall": - self.counters.uninst += 1 # filter out old-style virtual matches elif installed_versions and \ installed_versions[0].cp == pkg.cp: @@ -780,19 +765,31 @@ class Display(object): myinslotlist = None if myinslotlist: myoldbest = myinslotlist[:] - addl = self._insert_slot(pkg, pkg_info, myinslotlist) + if not cpvequal(pkg.cpv, + best([pkg.cpv] + [x.cpv for x in myinslotlist])): + # Downgrade in slot + pkg_info.attr_display.new_version = True + pkg_info.attr_display.downgrade = True + if pkg_info.ordered: + self.counters.downgrades += 1 + else: + # Update in slot + pkg_info.attr_display.new_version = True + if pkg_info.ordered: + self.counters.upgrades += 1 else: myoldbest = installed_versions - addl = self._new_slot(pkg, pkg_info) + pkg_info.attr_display.new = True + pkg_info.attr_display.new_slot = True + if pkg_info.ordered: + self.counters.newslot += 1 if self.conf.changelog: self.do_changelog(pkg, pkg_info) else: - addl = " " + green("N") + " " + pkg_info.fetch_symbol + " " + pkg_info.attr_display.new = True if pkg_info.ordered: self.counters.new += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl, myoldbest, myinslotlist + return myoldbest, myinslotlist def __call__(self, depgraph, mylist, favorites=None, verbosity=None): @@ -813,7 +810,7 @@ class Display(object): # files to fetch list - avoids counting a same file twice # in size display (verbose mode) self.myfetchlist = set() - + self.quiet_repo_display = "--quiet-repo-display" in depgraph._frozen_config.myopts if self.quiet_repo_display: # Use this set to detect when all the "repoadd" strings are "[0]" @@ -831,47 +828,52 @@ class Display(object): self.indent = " " * depth if isinstance(pkg, Blocker): - if self._blockers(pkg, fetch_symbol=" "): - continue + self._blockers(pkg) else: pkg_info = self.set_pkg_info(pkg, ordered) - addl, pkg_info.oldbest_list, myinslotlist = \ + pkg_info.oldbest_list, myinslotlist = \ self._get_installed_best(pkg, pkg_info) + if ordered and pkg_info.merge and \ + not pkg_info.attr_display.new: + for arg, atom in depgraph._iter_atoms_for_pkg(pkg): + if arg.force_reinstall: + pkg_info.attr_display.force_reinstall = True + break + self.verboseadd = "" if self.quiet_repo_display: self.repoadd = None - self._display_use(pkg, pkg_info.oldbest_list, myinslotlist) - self.recheck_hidden(pkg) + self._display_use(pkg, pkg_info) if self.conf.verbosity == 3: if self.quiet_repo_display: self.verbose_size(pkg, repoadd_set, pkg_info) else: self.verbose_size(pkg, None, pkg_info) - pkg_info.cp = pkg.cp - pkg_info.ver = self.get_ver_str(pkg) - self.oldlp = self.conf.columnwidth - 30 self.newlp = self.oldlp - 30 - pkg_info.oldbest = self.convert_myoldbest(pkg, pkg_info.oldbest_list) + pkg_info.oldbest = self.convert_myoldbest(pkg, pkg_info) pkg_info.system, pkg_info.world = \ self.check_system_world(pkg) - addl = self.set_interactive(pkg, pkg_info.ordered, addl) + if 'interactive' in pkg.properties and \ + pkg.operation == 'merge': + pkg_info.attr_display.interactive = True + if ordered: + self.counters.interactive += 1 if self.include_mask_str(): - addl += self.gen_mask_str(pkg) + pkg_info.attr_display.mask = self.gen_mask_str(pkg) if pkg.root_config.settings["ROOT"] != "/": if pkg_info.oldbest: pkg_info.oldbest += " " if self.conf.columns: - myprint = self._set_non_root_columns( - addl, pkg_info, pkg) + myprint = self._set_non_root_columns(pkg, pkg_info) else: pkg_str = pkg.cpv - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - pkg_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + pkg_str = self._append_slot(pkg_str, pkg, pkg_info) + pkg_str = self._append_repository(pkg_str, pkg, pkg_info) if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] " % ( @@ -880,17 +882,16 @@ class Display(object): ) else: myprint = "[%s %s] " % ( - self.pkgprint(pkg.type_name, pkg_info), addl) + self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display) myprint += self.indent + \ self.pkgprint(pkg_str, pkg_info) + " " + \ pkg_info.oldbest + darkgreen("to " + pkg.root) else: if self.conf.columns: - myprint = self._set_root_columns( - addl, pkg_info, pkg) + myprint = self._set_root_columns(pkg, pkg_info) else: - myprint = self._set_no_columns( - pkg, pkg_info, addl) + myprint = self._set_no_columns(pkg, pkg_info) if self.conf.columns and pkg.operation == "uninstall": continue diff --git a/portage_with_autodep/pym/_emerge/resolver/output.pyo b/portage_with_autodep/pym/_emerge/resolver/output.pyo Binary files differindex bd2ae2f..e3a3c7f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/output.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py index e751dd8..cfa6910 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py @@ -1,9 +1,12 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Contains private support functions for the Display class in output.py """ + +from __future__ import unicode_literals + __all__ = ( ) @@ -15,9 +18,10 @@ from portage import os from portage import _encodings, _unicode_encode from portage._sets.base import InternalPackageSet from portage.output import (blue, bold, colorize, create_color_func, - green, red, teal, yellow) + green, red, teal, turquoise, yellow) bad = create_color_func("BAD") from portage.util import shlex_split, writemsg +from portage.util.SlotObject import SlotObject from portage.versions import catpkgsplit from _emerge.Blocker import Blocker @@ -245,10 +249,9 @@ def _format_size(mysize): mystr=mystr[:mycount]+","+mystr[mycount:] return mystr+" kB" - def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, old_iuse, old_use, - is_new, reinst_flags): + is_new, feature_flags, reinst_flags): if not conf.print_use_string: return "" @@ -266,6 +269,7 @@ def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, any_iuse = cur_iuse.union(old_iuse) any_iuse = list(any_iuse) any_iuse.sort() + for flag in any_iuse: flag_str = None isEnabled = False @@ -299,7 +303,9 @@ def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, elif flag in old_use: flag_str = green("-" + flag) + "*" if flag_str: - if flag in iuse_forced: + if flag in feature_flags: + flag_str = "{" + flag_str + "}" + elif flag in iuse_forced: flag_str = "(" + flag_str + ")" if isEnabled: enabled.append(flag_str) @@ -611,9 +617,10 @@ class PkgInfo(object): information about the pkg being printed. """ - __slots__ = ("built", "cp", "ebuild_path", "fetch_symbol", "merge", - "oldbest", "oldbest_list", "operation", "ordered", - "repo_name", "repo_path_real", "system", "use", "ver", "world") + __slots__ = ("attr_display", "built", "cp", + "ebuild_path", "fetch_symbol", "merge", + "oldbest", "oldbest_list", "operation", "ordered", "previous_pkg", + "repo_name", "repo_path_real", "slot", "sub_slot", "system", "use", "ver", "world") def __init__(self): @@ -626,9 +633,74 @@ class PkgInfo(object): self.oldbest_list = [] self.operation = '' self.ordered = False + self.previous_pkg = None self.repo_path_real = '' self.repo_name = '' + self.slot = '' + self.sub_slot = '' self.system = False self.use = '' self.ver = '' self.world = False + self.attr_display = PkgAttrDisplay() + +class PkgAttrDisplay(SlotObject): + + __slots__ = ("downgrade", "fetch_restrict", "fetch_restrict_satisfied", + "force_reinstall", + "interactive", "mask", "new", "new_slot", "new_version", "replace") + + def __str__(self): + output = [] + + if self.interactive: + output.append(colorize("WARN", "I")) + else: + output.append(" ") + + if self.new or self.force_reinstall: + if self.force_reinstall: + output.append(red("r")) + else: + output.append(green("N")) + else: + output.append(" ") + + if self.new_slot or self.replace: + if self.replace: + output.append(yellow("R")) + else: + output.append(green("S")) + else: + output.append(" ") + + if self.fetch_restrict or self.fetch_restrict_satisfied: + if self.fetch_restrict_satisfied: + output.append(green("f")) + else: + output.append(red("F")) + else: + output.append(" ") + + if self.new_version: + output.append(turquoise("U")) + else: + output.append(" ") + + if self.downgrade: + output.append(blue("D")) + else: + output.append(" ") + + if self.mask is not None: + output.append(self.mask) + + return "".join(output) + + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content']) diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo Binary files differindex ae39dd4..225f4bc 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py index a1c8714..a193baa 100644 --- a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py @@ -1,10 +1,11 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals import sys +from portage import _encodings, _unicode_encode from _emerge.AtomArg import AtomArg from _emerge.Package import Package from _emerge.PackageArg import PackageArg @@ -150,7 +151,7 @@ class slot_conflict_handler(object): if self.debug: writemsg("\nNew configuration:\n", noiselevel=-1) for pkg in config: - writemsg(" " + str(pkg) + "\n", noiselevel=-1) + writemsg(" %s\n" % (pkg,), noiselevel=-1) writemsg("\n", noiselevel=-1) new_solutions = self._check_configuration(config, all_conflict_atoms_by_slotatom, conflict_nodes) @@ -225,10 +226,14 @@ class slot_conflict_handler(object): new_change = {} for pkg in solution: for flag, state in solution[pkg].items(): + real_flag = pkg.iuse.get_real_flag(flag) + if real_flag is None: + # Triggered by use-dep defaults. + continue if state == "enabled" and flag not in _pkg_use_enabled(pkg): - new_change.setdefault(pkg, {})[flag] = True + new_change.setdefault(pkg, {})[real_flag] = True elif state == "disabled" and flag in _pkg_use_enabled(pkg): - new_change.setdefault(pkg, {})[flag] = False + new_change.setdefault(pkg, {})[real_flag] = False return new_change def _prepare_conflict_msg_and_check_for_specificity(self): @@ -236,6 +241,7 @@ class slot_conflict_handler(object): Print all slot conflicts in a human readable way. """ _pkg_use_enabled = self.depgraph._pkg_use_enabled + verboseconflicts = "--verbose-conflicts" in self.myopts msg = self.conflict_msg indent = " " msg.append("\n!!! Multiple package instances within a single " + \ @@ -245,14 +251,14 @@ class slot_conflict_handler(object): for (slot_atom, root), pkgs \ in self.slot_collision_info.items(): - msg.append(str(slot_atom)) + msg.append("%s" % (slot_atom,)) if root != self.depgraph._frozen_config._running_root.root: msg.append(" for %s" % (root,)) msg.append("\n\n") for pkg in pkgs: msg.append(indent) - msg.append(str(pkg)) + msg.append("%s" % (pkg,)) parent_atoms = self.all_parents.get(pkg) if parent_atoms: #Create a list of collision reasons and map them to sets @@ -275,19 +281,29 @@ class slot_conflict_handler(object): if not atom_without_use_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): - #The version range does not match. - sub_type = None - if atom.operator in (">=", ">"): - sub_type = "ge" - elif atom.operator in ("=", "~"): - sub_type = "eq" - elif atom.operator in ("<=", "<"): - sub_type = "le" - - atoms = collision_reasons.get(("version", sub_type), set()) - atoms.add((ppkg, atom, other_pkg)) - num_all_specific_atoms += 1 - collision_reasons[("version", sub_type)] = atoms + if atom.operator is not None: + # The version range does not match. + sub_type = None + if atom.operator in (">=", ">"): + sub_type = "ge" + elif atom.operator in ("=", "~"): + sub_type = "eq" + elif atom.operator in ("<=", "<"): + sub_type = "le" + + key = ("version", sub_type) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + else: + # The sub_slot does not match. + key = ("sub-slot", atom.sub_slot) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + elif not atom_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): missing_iuse = other_pkg.iuse.get_missing_iuse( @@ -302,6 +318,26 @@ class slot_conflict_handler(object): #Use conditionals not met. violated_atom = atom.violated_conditionals(_pkg_use_enabled(other_pkg), \ other_pkg.iuse.is_valid_flag) + if violated_atom.use is None: + # Something like bug #453400 caused the + # above findAtomForPackage call to + # return None unexpectedly. + msg = ("\n\n!!! BUG: Detected " + "USE dep match inconsistency:\n" + "\tppkg: %s\n" + "\tviolated_atom: %s\n" + "\tatom: %s unevaluated: %s\n" + "\tother_pkg: %s IUSE: %s USE: %s\n" % + (ppkg, + violated_atom, + atom, + atom.unevaluated_atom, + other_pkg, + sorted(other_pkg.iuse.all), + sorted(_pkg_use_enabled(other_pkg)))) + writemsg(msg, noiselevel=-2) + raise AssertionError( + 'BUG: USE dep match inconsistency') for flag in violated_atom.use.enabled.union(violated_atom.use.disabled): atoms = collision_reasons.get(("use", flag), set()) atoms.add((ppkg, atom, other_pkg)) @@ -332,7 +368,14 @@ class slot_conflict_handler(object): best_matches[atom.cp] = (ppkg, atom) else: best_matches[atom.cp] = (ppkg, atom) - selected_for_display.update(best_matches.values()) + if verboseconflicts: + selected_for_display.add((ppkg, atom)) + if not verboseconflicts: + selected_for_display.update( + best_matches.values()) + elif type == "sub-slot": + for ppkg, atom, other_pkg in parents: + selected_for_display.add((ppkg, atom)) elif type == "use": #Prefer atoms with unconditional use deps over, because it's #not possible to change them on the parent, which means there @@ -377,7 +420,7 @@ class slot_conflict_handler(object): def highlight_violations(atom, version, use=[]): """Colorize parts of an atom""" - atom_str = str(atom) + atom_str = "%s" % (atom,) if version: op = atom.operator ver = None @@ -432,24 +475,27 @@ class slot_conflict_handler(object): (PackageArg, AtomArg)): # For PackageArg and AtomArg types, it's # redundant to display the atom attribute. - msg.append(str(parent)) + msg.append("%s" % (parent,)) else: # Display the specific atom from SetArg or # Package types. version_violated = False + sub_slot_violated = False use = [] for (type, sub_type), parents in collision_reasons.items(): for x in parents: if parent == x[0] and atom == x[1]: if type == "version": version_violated = True + elif type == "sub-slot": + sub_slot_violated = True elif type == "use": use.append(sub_type) break atom_str = highlight_violations(atom.unevaluated_atom, version_violated, use) - if version_violated: + if version_violated or sub_slot_violated: self.is_a_version_conflict = True msg.append("%s required by %s" % (atom_str, parent)) @@ -547,7 +593,9 @@ class slot_conflict_handler(object): if pkg.iuse.all.symmetric_difference(other_pkg.iuse.all) \ or _pkg_use_enabled(pkg).symmetric_difference(_pkg_use_enabled(other_pkg)): if self.debug: - writemsg(str(pkg) + " has pending USE changes. Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s has pending USE changes. " + "Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False #A list of dicts. Keeps one dict per slot conflict. [ { flag1: "enabled" }, { flag2: "disabled" } ] @@ -570,16 +618,18 @@ class slot_conflict_handler(object): if not i.findAtomForPackage(pkg, modified_use=_pkg_use_enabled(pkg)): #Version range does not match. if self.debug: - writemsg(str(pkg) + " does not satify all version requirements." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s does not satify all version " + "requirements. Rejecting configuration.\n") % + (pkg,), noiselevel=-1) return False if not pkg.iuse.is_valid_flag(atom.unevaluated_atom.use.required): #Missing IUSE. #FIXME: This needs to support use dep defaults. if self.debug: - writemsg(str(pkg) + " misses needed flags from IUSE." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s misses needed flags from IUSE." + " Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False if not isinstance(ppkg, Package) or ppkg.installed: @@ -604,8 +654,9 @@ class slot_conflict_handler(object): #We can't change USE of an installed package (only of an ebuild, but that is already #part of the conflict, isn't it? if self.debug: - writemsg(str(pkg) + ": installed package would need USE changes." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s: installed package would need USE" + " changes. Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False #Compute the required USE changes. A flag can be forced to "enabled" or "disabled", @@ -659,7 +710,7 @@ class slot_conflict_handler(object): if self.debug: writemsg("All involved flags:\n", noiselevel=-1) for id, involved_flags in enumerate(all_involved_flags): - writemsg(" " + str(config[id]) + "\n", noiselevel=-1) + writemsg(" %s\n" % (config[id],), noiselevel=-1) for flag, state in involved_flags.items(): writemsg(" " + flag + ": " + state + "\n", noiselevel=-1) @@ -742,7 +793,7 @@ class slot_conflict_handler(object): inner_first = False else: msg += ", " - msg += flag + ": " + str(state) + msg += flag + ": %s" % (state,) msg += "}" msg += "]\n" writemsg(msg, noiselevel=-1) @@ -846,8 +897,9 @@ class slot_conflict_handler(object): #We managed to create a new problem with our changes. is_valid_solution = False if self.debug: - writemsg("new conflict introduced: " + str(pkg) + \ - " does not match " + new_atom + " from " + str(ppkg) + "\n", noiselevel=-1) + writemsg(("new conflict introduced: %s" + " does not match %s from %s\n") % + (pkg, new_atom, ppkg), noiselevel=-1) break if not is_valid_solution: @@ -855,7 +907,7 @@ class slot_conflict_handler(object): #Make sure the changes don't violate REQUIRED_USE for pkg in required_changes: - required_use = pkg.metadata.get("REQUIRED_USE") + required_use = pkg._metadata.get("REQUIRED_USE") if not required_use: continue @@ -934,8 +986,16 @@ class _solution_candidate_generator(object): else: return self.value == other.value def __str__(self): - return str(self.value) - + return "%s" % (self.value,) + + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __init__(self, all_involved_flags): #A copy of all_involved_flags with all "cond" values #replaced by a _value_helper object. diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo Binary files differindex 1fc3a13..2ed98c5 100644 --- a/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo diff --git a/portage_with_autodep/pym/_emerge/search.py b/portage_with_autodep/pym/_emerge/search.py index 5abc8a0..bd74fb7 100644 --- a/portage_with_autodep/pym/_emerge/search.py +++ b/portage_with_autodep/pym/_emerge/search.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -69,7 +69,7 @@ class search(object): return db.aux_get(*args, **kwargs) except KeyError: pass - raise + raise KeyError(args[0]) def _findname(self, *args, **kwargs): for db in self._dbs: diff --git a/portage_with_autodep/pym/_emerge/search.pyo b/portage_with_autodep/pym/_emerge/search.pyo Binary files differindex 055a734..51eef02 100644 --- a/portage_with_autodep/pym/_emerge/search.pyo +++ b/portage_with_autodep/pym/_emerge/search.pyo diff --git a/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo Binary files differindex 337e135..4b3d724 100644 --- a/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo +++ b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo diff --git a/portage_with_autodep/pym/_emerge/stdout_spinner.py b/portage_with_autodep/pym/_emerge/stdout_spinner.py index 5ad31f0..670686a 100644 --- a/portage_with_autodep/pym/_emerge/stdout_spinner.py +++ b/portage_with_autodep/pym/_emerge/stdout_spinner.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import platform @@ -53,17 +53,18 @@ class stdout_spinner(object): def update_basic(self): self.spinpos = (self.spinpos + 1) % 500 if self._return_early(): - return + return True if (self.spinpos % 100) == 0: if self.spinpos == 0: sys.stdout.write(". ") else: sys.stdout.write(".") sys.stdout.flush() + return True def update_scroll(self): if self._return_early(): - return + return True if(self.spinpos >= len(self.scroll_sequence)): sys.stdout.write(darkgreen(" \b\b\b" + self.scroll_sequence[ len(self.scroll_sequence) - 1 - (self.spinpos % len(self.scroll_sequence))])) @@ -71,13 +72,15 @@ class stdout_spinner(object): sys.stdout.write(green("\b " + self.scroll_sequence[self.spinpos])) sys.stdout.flush() self.spinpos = (self.spinpos + 1) % (2 * len(self.scroll_sequence)) + return True def update_twirl(self): self.spinpos = (self.spinpos + 1) % len(self.twirl_sequence) if self._return_early(): - return + return True sys.stdout.write("\b\b " + self.twirl_sequence[self.spinpos]) sys.stdout.flush() + return True def update_quiet(self): - return + return True diff --git a/portage_with_autodep/pym/_emerge/stdout_spinner.pyo b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo Binary files differindex b091171..d7a93b1 100644 --- a/portage_with_autodep/pym/_emerge/stdout_spinner.pyo +++ b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/__init__.pyo b/portage_with_autodep/pym/_emerge/sync/__init__.pyo Binary files differindex 7314f80..4163c1a 100644 --- a/portage_with_autodep/pym/_emerge/sync/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/sync/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo Binary files differindex 8e41377..9dba228 100644 --- a/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo +++ b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo Binary files differindex 5b59c5a..e5d4588 100644 --- a/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo +++ b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo diff --git a/portage_with_autodep/pym/_emerge/unmerge.pyo b/portage_with_autodep/pym/_emerge/unmerge.pyo Binary files differindex c16c9ec..c0cb01b 100644 --- a/portage_with_autodep/pym/_emerge/unmerge.pyo +++ b/portage_with_autodep/pym/_emerge/unmerge.pyo diff --git a/portage_with_autodep/pym/_emerge/userquery.pyo b/portage_with_autodep/pym/_emerge/userquery.pyo Binary files differindex 5492f90..5c2fc9e 100644 --- a/portage_with_autodep/pym/_emerge/userquery.pyo +++ b/portage_with_autodep/pym/_emerge/userquery.pyo diff --git a/portage_with_autodep/pym/portage/__init__.py b/portage_with_autodep/pym/portage/__init__.py index 431dc26..2bce5d7 100644 --- a/portage_with_autodep/pym/portage/__init__.py +++ b/portage_with_autodep/pym/portage/__init__.py @@ -1,8 +1,10 @@ # portage.py -- core Portage functionality -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -VERSION="2.2.0_alpha108" +from __future__ import unicode_literals + +VERSION="2.2.8-r1" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -16,14 +18,6 @@ try: errno.ESTALE = -1 import re import types - - # Try the commands module first, since this allows us to eliminate - # the subprocess module from the baseline imports under python2. - try: - from commands import getstatusoutput as subprocess_getstatusoutput - except ImportError: - from subprocess import getstatusoutput as subprocess_getstatusoutput - import platform # Temporarily delete these imports, to ensure that only the @@ -70,6 +64,7 @@ try: 'match_from_list,match_to_list', 'portage.dep.dep_check:dep_check,dep_eval,dep_wordreduce,dep_zapdeps', 'portage.eclass_cache', + 'portage.elog', 'portage.exception', 'portage.getbinpkg', 'portage.locks', @@ -114,6 +109,7 @@ try: 'cpv_getkey@getCPFromCPV,endversion_keys,' + \ 'suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify', 'portage.xpak', + 'subprocess', 'time', ) @@ -178,6 +174,15 @@ _encodings = { } if sys.hexversion >= 0x3000000: + + def _decode_argv(argv): + # With Python 3, the surrogateescape encoding error handler makes it + # possible to access the original argv bytes, which can be useful + # if their actual encoding does no match the filesystem encoding. + fs_encoding = sys.getfilesystemencoding() + return [_unicode_decode(x.encode(fs_encoding, 'surrogateescape')) + for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, str): s = s.encode(encoding, errors) @@ -187,7 +192,13 @@ if sys.hexversion >= 0x3000000: if isinstance(s, bytes): s = str(s, encoding=encoding, errors=errors) return s + + _native_string = _unicode_decode else: + + def _decode_argv(argv): + return [_unicode_decode(x) for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, unicode): s = s.encode(encoding, errors) @@ -198,6 +209,17 @@ else: s = unicode(s, encoding=encoding, errors=errors) return s + _native_string = _unicode_encode + +if sys.hexversion >= 0x20605f0: + def _native_kwargs(kwargs): + return kwargs +else: + # Avoid "TypeError: keywords must be strings" issue triggered + # by unicode_literals: http://bugs.python.org/issue4978 + def _native_kwargs(kwargs): + return dict((_native_string(k), v) for k, v in kwargs.iteritems()) + class _unicode_func_wrapper(object): """ Wraps a function, converts arguments from unicode to bytes, @@ -215,7 +237,7 @@ class _unicode_func_wrapper(object): self._func = func self._encoding = encoding - def __call__(self, *args, **kwargs): + def _process_args(self, args, kwargs): encoding = self._encoding wrapped_args = [_unicode_encode(x, encoding=encoding, errors='strict') @@ -227,6 +249,13 @@ class _unicode_func_wrapper(object): else: wrapped_kwargs = {} + return (wrapped_args, wrapped_kwargs) + + def __call__(self, *args, **kwargs): + + encoding = self._encoding + wrapped_args, wrapped_kwargs = self._process_args(args, kwargs) + rval = self._func(*wrapped_args, **wrapped_kwargs) # Don't use isinstance() since we don't want to convert subclasses @@ -294,12 +323,17 @@ class _unicode_module_wrapper(object): import os as _os _os_overrides = { id(_os.fdopen) : _os.fdopen, - id(_os.mkfifo) : _os.mkfifo, id(_os.popen) : _os.popen, id(_os.read) : _os.read, id(_os.system) : _os.system, } + +try: + _os_overrides[id(_os.mkfifo)] = _os.mkfifo +except AttributeError: + pass # Jython + if hasattr(_os, 'statvfs'): _os_overrides[id(_os.statvfs)] = _os.statvfs @@ -334,16 +368,25 @@ except (ImportError, OSError) as e: _python_interpreter = os.path.realpath(sys.executable) _bin_path = PORTAGE_BIN_PATH _pym_path = PORTAGE_PYM_PATH +_working_copy = VERSION == "HEAD" + +# Api consumers included in portage should set this to True. +_internal_caller = False -if sys.hexversion >= 0x3030000: - # Workaround for http://bugs.python.org/issue14007 - def _test_xml_etree_ElementTree_TreeBuilder_type(): - import subprocess - p = subprocess.Popen([_python_interpreter, "-c", - "import sys, xml.etree.ElementTree; sys.exit(not isinstance(xml.etree.ElementTree.TreeBuilder, type))"]) - if p.wait() != 0: - sys.modules["_elementtree"] = None - _test_xml_etree_ElementTree_TreeBuilder_type() +_sync_disabled_warnings = False + +def _get_stdin(): + """ + Buggy code in python's multiprocessing/process.py closes sys.stdin + and reassigns it to open(os.devnull), but fails to update the + corresponding __stdin__ reference. So, detect that case and handle + it appropriately. + """ + if not sys.__stdin__.closed: + return sys.__stdin__ + return sys.stdin + +_shell_quote_re = re.compile(r"[\s><=*\\\"'$`]") def _shell_quote(s): """ @@ -351,6 +394,8 @@ def _shell_quote(s): escape any backslashes, double-quotes, dollar signs, or backquotes in the string. """ + if _shell_quote_re.search(s) is None: + return s for letter in "\\\"$`": if letter in s: s = s.replace(letter, "\\" + letter) @@ -364,8 +409,27 @@ if platform.system() in ('FreeBSD',): @classmethod def chflags(cls, path, flags, opts=""): - cmd = 'chflags %s %o %s' % (opts, flags, _shell_quote(path)) - status, output = subprocess_getstatusoutput(cmd) + cmd = ['chflags'] + if opts: + cmd.append(opts) + cmd.append('%o' % (flags,)) + cmd.append(path) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = process.find_binary(cmd[0]) + if fullname is None: + raise exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + encoding = _encodings['fs'] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = proc.communicate()[0] + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: return # Try to generate an ENOENT error if appropriate. @@ -378,6 +442,7 @@ if platform.system() in ('FreeBSD',): raise portage.exception.CommandNotFound('chflags') # Now we're not sure exactly why it failed or what # the real errno was, so just report EPERM. + output = _unicode_decode(output, encoding=encoding) e = OSError(errno.EPERM, output) e.errno = errno.EPERM e.filename = path @@ -406,7 +471,15 @@ def getcwd(): getcwd() def abssymlink(symlink, target=None): - "This reads symlinks, resolving the relative symlinks, and returning the absolute." + """ + This reads symlinks, resolving the relative symlinks, + and returning the absolute. + @param symlink: path of symlink (must be absolute) + @param target: the target of the symlink (as returned + by readlink) + @rtype: str + @return: the absolute path of the symlink target + """ if target is not None: mylink = target else: @@ -418,8 +491,9 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python"]) -_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend"]) +_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2"]) +_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI)] + list(_testing_eapis) + list(_deprecated_eapis)) def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis @@ -476,14 +550,13 @@ auxdbkeys = ( 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'PDEPEND', 'PROVIDE', 'EAPI', - 'PROPERTIES', 'DEFINED_PHASES', 'UNUSED_05', 'UNUSED_04', + 'PROPERTIES', 'DEFINED_PHASES', 'HDEPEND', 'UNUSED_04', 'UNUSED_03', 'UNUSED_02', 'UNUSED_01', ) auxdbkeylen=len(auxdbkeys) def portageexit(): - if data.secpass > 1 and os.environ.get("SANDBOX_ON") != "1": - close_portdbapi_caches() + pass class _trees_dict(dict): __slots__ = ('_running_eroot', '_target_eroot',) @@ -494,13 +567,6 @@ class _trees_dict(dict): def create_trees(config_root=None, target_root=None, trees=None, env=None, eprefix=None): - if trees is not None: - # clean up any existing portdbapi instances - for myroot in trees: - portdb = trees[myroot]["porttree"].dbapi - portdb.close_caches() - portdbapi.portdbapi_instances.remove(portdb) - del trees[myroot]["porttree"], myroot, portdb if trees is None: trees = _trees_dict() @@ -518,7 +584,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, trees._target_eroot = settings['EROOT'] myroots = [(settings['EROOT'], settings)] - if settings["ROOT"] == "/": + if settings["ROOT"] == "/" and settings["EPREFIX"] == const.EPREFIX: trees._running_eroot = trees._target_eroot else: @@ -526,15 +592,15 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, # environment to apply to the config that's associated # with ROOT != "/", so pass a nearly empty dict for the env parameter. clean_env = {} - for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_USERNAME', - 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', + for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_REPOSITORIES', 'PORTAGE_USERNAME', + 'PYTHONPATH', 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', 'ftp_proxy', 'http_proxy', 'no_proxy', '__PORTAGE_TEST_HARDLINK_LOCKS'): v = settings.get(k) if v is not None: clean_env[k] = v settings = config(config_root=None, target_root="/", - env=clean_env, eprefix=eprefix) + env=clean_env, eprefix=None) settings.lock() trees._running_eroot = settings['EROOT'] myroots.append((settings['EROOT'], settings)) @@ -558,11 +624,17 @@ if VERSION == 'HEAD': if VERSION is not self: return VERSION if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, '.git')): - status, output = subprocess_getstatusoutput(( - "cd %s ; git describe --tags || exit $? ; " + \ + encoding = _encodings['fs'] + cmd = [BASH_BINARY, "-c", ("cd %s ; git describe --tags || exit $? ; " + \ "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \ "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \ - "exit 0") % _shell_quote(PORTAGE_BASE_PATH)) + "exit 0") % _shell_quote(PORTAGE_BASE_PATH)] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], encoding=encoding) + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: output_lines = output.splitlines() if output_lines: @@ -591,34 +663,17 @@ if VERSION == 'HEAD': return VERSION VERSION = _LazyVersion() -if "_legacy_globals_constructed" in globals(): - # The module has been reloaded, so perform any relevant cleanup - # and prevent memory leaks. - if "db" in _legacy_globals_constructed: - try: - db - except NameError: - pass - else: - if isinstance(db, dict) and db: - for _x in db.values(): - try: - if "porttree" in _x.lazy_items: - continue - except (AttributeError, TypeError): - continue - try: - _x = _x["porttree"].dbapi - except (AttributeError, KeyError): - continue - if not isinstance(_x, portdbapi): - continue - _x.close_caches() - try: - portdbapi.portdbapi_instances.remove(_x) - except ValueError: - pass - del _x +_legacy_global_var_names = ("archlist", "db", "features", + "groups", "mtimedb", "mtimedbfile", "pkglines", + "portdb", "profiledir", "root", "selinux_enabled", + "settings", "thirdpartymirrors") + +def _reset_legacy_globals(): + + global _legacy_globals_constructed + _legacy_globals_constructed = set() + for k in _legacy_global_var_names: + globals()[k] = _LegacyGlobalProxy(k) class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): @@ -633,16 +688,7 @@ class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): from portage._legacy_globals import _get_legacy_global return _get_legacy_global(name) -_legacy_global_var_names = ("archlist", "db", "features", - "groups", "mtimedb", "mtimedbfile", "pkglines", - "portdb", "profiledir", "root", "selinux_enabled", - "settings", "thirdpartymirrors") - -for k in _legacy_global_var_names: - globals()[k] = _LegacyGlobalProxy(k) -del k - -_legacy_globals_constructed = set() +_reset_legacy_globals() def _disable_legacy_globals(): """ diff --git a/portage_with_autodep/pym/portage/__init__.pyo b/portage_with_autodep/pym/portage/__init__.pyo Binary files differindex 9fc449e..ef60b98 100644 --- a/portage_with_autodep/pym/portage/__init__.pyo +++ b/portage_with_autodep/pym/portage/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_global_updates.py b/portage_with_autodep/pym/portage/_global_updates.py index 5175043..9ae734b 100644 --- a/portage_with_autodep/pym/portage/_global_updates.py +++ b/portage_with_autodep/pym/portage/_global_updates.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -32,22 +32,20 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): """ # only do this if we're root and not running repoman/ebuild digest - retupd = False if secpass < 2 or \ "SANDBOX_ACTIVE" in os.environ or \ len(trees) != 1: - return retupd + return False + + return _do_global_updates(trees, prev_mtimes, + quiet=quiet, if_mtime_changed=if_mtime_changed) + +def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): root = trees._running_eroot mysettings = trees[root]["vartree"].settings portdb = trees[root]["porttree"].dbapi vardb = trees[root]["vartree"].dbapi bindb = trees[root]["bintree"].dbapi - if not os.access(bindb.bintree.pkgdir, os.W_OK): - bindb = None - else: - # Call binarytree.populate(), since we want to make sure it's - # only populated with local packages here (getbinpkgs=0). - bindb.bintree.populate() world_file = os.path.join(mysettings['EROOT'], WORLD_FILE) world_list = grabfile(world_file) @@ -61,6 +59,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): repo_map = {} timestamps = {} + retupd = False update_notice_printed = False for repo_name in portdb.getRepositories(): repo = portdb.getRepositoryPath(repo_name) @@ -115,6 +114,14 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): if myupd: retupd = True + if retupd: + if os.access(bindb.bintree.pkgdir, os.W_OK): + # Call binarytree.populate(), since we want to make sure it's + # only populated with local packages here (getbinpkgs=0). + bindb.bintree.populate() + else: + bindb = None + master_repo = portdb.getRepositoryName(portdb.porttree_root) if master_repo in repo_map: repo_map['DEFAULT'] = repo_map[master_repo] @@ -237,8 +244,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): # Update progress above is indicated by characters written to stdout so # we print a couple new lines here to separate the progress output from # what follows. - print() - print() + writemsg_stdout("\n\n") if do_upgrade_packagesmessage and bindb and \ bindb.cpv_all(): diff --git a/portage_with_autodep/pym/portage/_global_updates.pyo b/portage_with_autodep/pym/portage/_global_updates.pyo Binary files differindex 3e2e8de..fe5684e 100644 --- a/portage_with_autodep/pym/portage/_global_updates.pyo +++ b/portage_with_autodep/pym/portage/_global_updates.pyo diff --git a/portage_with_autodep/pym/portage/_legacy_globals.py b/portage_with_autodep/pym/portage/_legacy_globals.py index abffa0e..bb9691a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.py +++ b/portage_with_autodep/pym/portage/_legacy_globals.py @@ -27,7 +27,8 @@ def _get_legacy_global(name): os.umask(0o22) kwargs = {} - for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): + for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), + ("target_root", "ROOT"), ("eprefix", "EPREFIX")): kwargs[k] = os.environ.get(envvar) portage._initializing_globals = True diff --git a/portage_with_autodep/pym/portage/_legacy_globals.pyo b/portage_with_autodep/pym/portage/_legacy_globals.pyo Binary files differindex 2e50cbe..2a5d08a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.pyo +++ b/portage_with_autodep/pym/portage/_legacy_globals.pyo diff --git a/portage_with_autodep/pym/portage/_selinux.py b/portage_with_autodep/pym/portage/_selinux.py index 9470978..e4621b1 100644 --- a/portage_with_autodep/pym/portage/_selinux.py +++ b/portage_with_autodep/pym/portage/_selinux.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Don't use the unicode-wrapped os and shutil modules here since @@ -8,18 +8,18 @@ import shutil import portage from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode +from portage import _native_string, _unicode_decode from portage.localization import _ portage.proxy.lazyimport.lazyimport(globals(), 'selinux') def copyfile(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("copyfile: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -39,12 +39,12 @@ def is_selinux_enabled(): return selinux.is_selinux_enabled() def mkdir(target, refdir): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + refdir = _native_string(refdir, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.getfilecon(refdir) if rc < 0: - refdir = _unicode_decode(refdir, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + refdir = _unicode_decode(refdir, encoding=_encodings['fs'], errors='replace') raise OSError( _("mkdir: Failed getting context of reference directory \"%s\".") \ % refdir) @@ -56,11 +56,12 @@ def mkdir(target, refdir): setfscreate() def rename(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("rename: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -75,10 +76,10 @@ def settype(newtype): return ":".join(ret) def setexec(ctx="\n"): - ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setexeccon(ctx) < 0: - ctx = _unicode_decode(ctx, encoding=_encodings['content'], - errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') if selinux.security_getenforce() == 1: raise OSError(_("Failed setting exec() context \"%s\".") % ctx) else: @@ -87,37 +88,47 @@ def setexec(ctx="\n"): noiselevel=-1) def setfscreate(ctx="\n"): - ctx = _unicode_encode(ctx, - encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setfscreatecon(ctx) < 0: - ctx = _unicode_decode(ctx, - encoding=_encodings['content'], errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) -def spawn_wrapper(spawn_func, selinux_type): +class spawn_wrapper(object): + """ + Create a wrapper function for the given spawn function. When the wrapper + is called, it will adjust the arguments such that setexec() to be called + *after* the fork (thereby avoiding any interference with concurrent + threads in the calling process). + """ + __slots__ = ("_con", "_spawn_func") - selinux_type = _unicode_encode(selinux_type, - encoding=_encodings['content'], errors='strict') + def __init__(self, spawn_func, selinux_type): + self._spawn_func = spawn_func + selinux_type = _native_string(selinux_type, encoding=_encodings['content'], errors='strict') + self._con = settype(selinux_type) - def wrapper_func(*args, **kwargs): - con = settype(selinux_type) - setexec(con) - try: - return spawn_func(*args, **kwargs) - finally: - setexec() + def __call__(self, *args, **kwargs): - return wrapper_func + pre_exec = kwargs.get("pre_exec") + + def _pre_exec(): + if pre_exec is not None: + pre_exec() + setexec(self._con) + + kwargs["pre_exec"] = _pre_exec + return self._spawn_func(*args, **kwargs) def symlink(target, link, reflnk): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - link = _unicode_encode(link, encoding=_encodings['fs'], errors='strict') - reflnk = _unicode_encode(reflnk, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + link = _native_string(link, encoding=_encodings['fs'], errors='strict') + reflnk = _native_string(reflnk, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(reflnk) if rc < 0: - reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], errors='replace') raise OSError( _("symlink: Failed getting context of reference symlink \"%s\".") \ % reflnk) diff --git a/portage_with_autodep/pym/portage/_selinux.pyo b/portage_with_autodep/pym/portage/_selinux.pyo Binary files differindex 7a413e0..080613d 100644 --- a/portage_with_autodep/pym/portage/_selinux.pyo +++ b/portage_with_autodep/pym/portage/_selinux.pyo diff --git a/portage_with_autodep/pym/portage/_sets/__init__.py b/portage_with_autodep/pym/portage/_sets/__init__.py index 88a4b3b..75d1df7 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.py +++ b/portage_with_autodep/pym/portage/_sets/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -17,6 +17,7 @@ try: from configparser import SafeConfigParser except ImportError: from ConfigParser import SafeConfigParser, NoOptionError, ParsingError +import portage from portage import os from portage import load_mod from portage import _unicode_decode @@ -111,16 +112,51 @@ class SetConfig(object): """ parser = self._parser + parser.remove_section("world") parser.add_section("world") parser.set("world", "class", "portage.sets.base.DummyPackageSet") parser.set("world", "packages", "@selected @system") + parser.remove_section("selected") parser.add_section("selected") parser.set("selected", "class", "portage.sets.files.WorldSelectedSet") + parser.remove_section("system") parser.add_section("system") parser.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + parser.remove_section("security") + parser.add_section("security") + parser.set("security", "class", "portage.sets.security.NewAffectedSet") + + parser.remove_section("usersets") + parser.add_section("usersets") + parser.set("usersets", "class", "portage.sets.files.StaticFileSet") + parser.set("usersets", "multiset", "true") + parser.set("usersets", "directory", "%(PORTAGE_CONFIGROOT)setc/portage/sets") + parser.set("usersets", "world-candidate", "true") + + parser.remove_section("live-rebuild") + parser.add_section("live-rebuild") + parser.set("live-rebuild", "class", "portage.sets.dbapi.VariableSet") + parser.set("live-rebuild", "variable", "INHERITED") + parser.set("live-rebuild", "includes", " ".join(sorted(portage.const.LIVE_ECLASSES))) + + parser.remove_section("module-rebuild") + parser.add_section("module-rebuild") + parser.set("module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("module-rebuild", "files", "/lib/modules") + + parser.remove_section("preserved-rebuild") + parser.add_section("preserved-rebuild") + parser.set("preserved-rebuild", "class", "portage.sets.libs.PreservedLibraryConsumerSet") + + parser.remove_section("x11-module-rebuild") + parser.add_section("x11-module-rebuild") + parser.set("x11-module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("x11-module-rebuild", "files", "/usr/lib/xorg/modules") + parser.set("x11-module-rebuild", "exclude-files", "/usr/bin/Xorg") + def update(self, setname, options): parser = self._parser self.errors = [] @@ -260,8 +296,8 @@ def load_default_config(settings, trees): return SetConfig(None, settings, trees) global_config_path = GLOBAL_CONFIG_PATH - if settings['EPREFIX']: - global_config_path = os.path.join(settings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) def _getfiles(): for path, dirs, files in os.walk(os.path.join(global_config_path, "sets")): diff --git a/portage_with_autodep/pym/portage/_sets/__init__.pyo b/portage_with_autodep/pym/portage/_sets/__init__.pyo Binary files differindex 5318dbe..500a5d3 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.pyo +++ b/portage_with_autodep/pym/portage/_sets/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_sets/base.py b/portage_with_autodep/pym/portage/_sets/base.py index c8d3ae4..d368e00 100644 --- a/portage_with_autodep/pym/portage/_sets/base.py +++ b/portage_with_autodep/pym/portage/_sets/base.py @@ -126,7 +126,7 @@ class PackageSet(object): if modified_use is not None and modified_use is not pkg.use.enabled: pkg = pkg.copy() - pkg.metadata["USE"] = " ".join(modified_use) + pkg._metadata["USE"] = " ".join(modified_use) # Atoms matched via PROVIDE must be temporarily transformed since # match_from_list() only works correctly when atom.cp == pkg.cp. @@ -156,7 +156,7 @@ class PackageSet(object): for atom in atoms: if match_from_list(atom, cpv_slot_list): yield atom - provides = pkg.metadata['PROVIDE'] + provides = pkg._metadata['PROVIDE'] if not provides: return provides = provides.split() diff --git a/portage_with_autodep/pym/portage/_sets/base.pyo b/portage_with_autodep/pym/portage/_sets/base.pyo Binary files differindex 89e53be..09ade2f 100644 --- a/portage_with_autodep/pym/portage/_sets/base.pyo +++ b/portage_with_autodep/pym/portage/_sets/base.pyo diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.py b/portage_with_autodep/pym/portage/_sets/dbapi.py index 4982a92..384fb3a 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.py +++ b/portage_with_autodep/pym/portage/_sets/dbapi.py @@ -26,8 +26,7 @@ class EverythingSet(PackageSet): def load(self): myatoms = [] - db_keys = ["SLOT"] - aux_get = self._db.aux_get + pkg_str = self._db._pkg_str cp_list = self._db.cp_list for cp in self._db.cp_all(): @@ -35,8 +34,8 @@ class EverythingSet(PackageSet): # NOTE: Create SLOT atoms even when there is only one # SLOT installed, in order to avoid the possibility # of unwanted upgrades as reported in bug #338959. - slot, = aux_get(cpv, db_keys) - atom = Atom("%s:%s" % (cp, slot)) + pkg = pkg_str(cpv, None) + atom = Atom("%s:%s" % (pkg.cp, pkg.slot)) if self._filter: if self._filter(atom): myatoms.append(atom) @@ -68,20 +67,19 @@ class OwnerSet(PackageSet): """ rValue = set() vardb = self._db - aux_get = vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = vardb._pkg_str if exclude_paths is None: for link, p in vardb._owners.iter_owners(paths): - slot, = aux_get(link.mycpv, aux_keys) - rValue.add("%s:%s" % (link.mycpv.cp, slot)) + pkg = pkg_str(link.mycpv, None) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) else: all_paths = set() all_paths.update(paths) all_paths.update(exclude_paths) exclude_atoms = set() for link, p in vardb._owners.iter_owners(all_paths): - slot, = aux_get(link.mycpv, aux_keys) - atom = "%s:%s" % (link.mycpv.cp, slot) + pkg = pkg_str(link.mycpv, None) + atom = "%s:%s" % (pkg.cp, pkg.slot) rValue.add(atom) if p in exclude_paths: exclude_atoms.add(atom) @@ -173,12 +171,11 @@ class DowngradeSet(PackageSet): xmatch = self._portdb.xmatch xmatch_level = "bestmatch-visible" cp_list = self._vardb.cp_list - aux_get = self._vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = self._vardb._pkg_str for cp in self._vardb.cp_all(): for cpv in cp_list(cp): - slot, = aux_get(cpv, aux_keys) - slot_atom = "%s:%s" % (cp, slot) + pkg = pkg_str(cpv, None) + slot_atom = "%s:%s" % (pkg.cp, pkg.slot) ebuild = xmatch(xmatch_level, slot_atom) if not ebuild: continue @@ -326,6 +323,7 @@ class CategorySet(PackageSet): class AgeSet(EverythingSet): _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) def __init__(self, vardb, mode="older", age=7): super(AgeSet, self).__init__(vardb) @@ -335,8 +333,12 @@ class AgeSet(EverythingSet): def _filter(self, atom): cpv = self._db.match(atom)[0] - path = self._db.getpath(cpv, filename="COUNTER") - age = (time.time() - os.stat(path).st_mtime) / (3600 * 24) + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + age = (time.time() - date) / (3600 * 24) if ((self._mode == "older" and age <= self._age) \ or (self._mode == "newer" and age >= self._age)): return False @@ -355,6 +357,83 @@ class AgeSet(EverythingSet): singleBuilder = classmethod(singleBuilder) +class DateSet(EverythingSet): + _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) + + def __init__(self, vardb, date, mode="older"): + super(DateSet, self).__init__(vardb) + self._mode = mode + self._date = date + + def _filter(self, atom): + + cpv = self._db.match(atom)[0] + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + # Make sure inequality is _strict_ to exclude tested package + if ((self._mode == "older" and date < self._date) \ + or (self._mode == "newer" and date > self._date)): + return True + else: + return False + + def singleBuilder(cls, options, settings, trees): + vardbapi = trees["vartree"].dbapi + mode = options.get("mode", "older") + if str(mode).lower() not in ["newer", "older"]: + raise SetConfigError(_("invalid 'mode' value %s (use either 'newer' or 'older')") % mode) + + formats = [] + if options.get("package") is not None: + formats.append("package") + if options.get("filestamp") is not None: + formats.append("filestamp") + if options.get("seconds") is not None: + formats.append("seconds") + if options.get("date") is not None: + formats.append("date") + + if not formats: + raise SetConfigError(_("none of these options specified: 'package', 'filestamp', 'seconds', 'date'")) + elif len(formats) > 1: + raise SetConfigError(_("no more than one of these options is allowed: 'package', 'filestamp', 'seconds', 'date'")) + + format = formats[0] + + if (format == "package"): + package = options.get("package") + try: + cpv = vardbapi.match(package)[0] + date, = vardbapi.aux_get(cpv, ('BUILD_TIME',)) + date = int(date) + except (KeyError, ValueError): + raise SetConfigError(_("cannot determine installation date of package %s") % package) + elif (format == "filestamp"): + filestamp = options.get("filestamp") + try: + date = int(os.stat(filestamp).st_mtime) + except (OSError, ValueError): + raise SetConfigError(_("cannot determine 'filestamp' of '%s'") % filestamp) + elif (format == "seconds"): + try: + date = int(options.get("seconds")) + except ValueError: + raise SetConfigError(_("option 'seconds' must be an integer")) + else: + dateopt = options.get("date") + try: + dateformat = options.get("dateformat", "%x %X") + date = int(time.mktime(time.strptime(dateopt, dateformat))) + except ValueError: + raise SetConfigError(_("'date=%s' does not match 'dateformat=%s'") % (dateopt, dateformat)) + return DateSet(vardb=vardbapi, date=date, mode=mode) + + singleBuilder = classmethod(singleBuilder) + class RebuiltBinaries(EverythingSet): _operations = ('merge',) _aux_keys = ('BUILD_TIME',) diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.pyo b/portage_with_autodep/pym/portage/_sets/dbapi.pyo Binary files differindex 20bf848..584cce4 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.pyo +++ b/portage_with_autodep/pym/portage/_sets/dbapi.pyo diff --git a/portage_with_autodep/pym/portage/_sets/files.py b/portage_with_autodep/pym/portage/_sets/files.py index f19ecf6..2fb64de 100644 --- a/portage_with_autodep/pym/portage/_sets/files.py +++ b/portage_with_autodep/pym/portage/_sets/files.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -11,7 +11,6 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path from portage.const import USER_CONFIG_PATH, WORLD_FILE, WORLD_SETS_FILE -from portage.const import _ENABLE_SET_CONFIG from portage.localization import _ from portage.locks import lockfile, unlockfile from portage import portage_gid @@ -87,8 +86,8 @@ class StaticFileSet(EditablePackageSet): for a in data: matches = self.dbapi.match(a) for cpv in matches: - atoms.append("%s:%s" % (cpv_getkey(cpv), - self.dbapi.aux_get(cpv, ["SLOT"])[0])) + pkg = self.dbapi._pkg_str(cpv, None) + atoms.append("%s:%s" % (pkg.cp, pkg.slot)) # In addition to any installed slots, also try to pull # in the latest new slot that may be available. atoms.append(a) @@ -231,9 +230,8 @@ class WorldSelectedSet(EditablePackageSet): write_atomic(self._filename, "".join(sorted("%s\n" % x for x in self._atoms))) - if _ENABLE_SET_CONFIG: - write_atomic(self._filename2, - "".join(sorted("%s\n" % x for x in self._nonatoms))) + write_atomic(self._filename2, + "".join(sorted("%s\n" % x for x in self._nonatoms))) def load(self): atoms = [] @@ -263,9 +261,8 @@ class WorldSelectedSet(EditablePackageSet): else: atoms.extend(self._atoms) - if _ENABLE_SET_CONFIG: - changed2, nonatoms = self._load2() - atoms_changed |= changed2 + changed2, nonatoms = self._load2() + atoms_changed |= changed2 if atoms_changed: self._setAtoms(atoms+nonatoms) @@ -299,10 +296,14 @@ class WorldSelectedSet(EditablePackageSet): ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=0o2750, mask=0o2) def lock(self): + if self._lock is not None: + raise AssertionError("already locked") self._ensure_dirs() self._lock = lockfile(self._filename, wantnewlockfile=1) def unlock(self): + if self._lock is None: + raise AssertionError("not locked") unlockfile(self._lock) self._lock = None diff --git a/portage_with_autodep/pym/portage/_sets/files.pyo b/portage_with_autodep/pym/portage/_sets/files.pyo Binary files differindex eb03c00..f79345b 100644 --- a/portage_with_autodep/pym/portage/_sets/files.pyo +++ b/portage_with_autodep/pym/portage/_sets/files.pyo diff --git a/portage_with_autodep/pym/portage/_sets/libs.py b/portage_with_autodep/pym/portage/_sets/libs.py index 6c5babc..022e076 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.py +++ b/portage_with_autodep/pym/portage/_sets/libs.py @@ -1,12 +1,12 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +from portage.exception import InvalidData from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import get_boolean, SetConfigError -from portage.versions import cpv_getkey import portage class LibraryConsumerSet(PackageSet): @@ -22,14 +22,14 @@ class LibraryConsumerSet(PackageSet): for p in paths: for cpv in self.dbapi._linkmap.getOwners(p): try: - slot, = self.dbapi.aux_get(cpv, ["SLOT"]) - except KeyError: + pkg = self.dbapi._pkg_str(cpv, None) + except (KeyError, InvalidData): # This is expected for preserved libraries # of packages that have been uninstalled # without replacement. pass else: - rValue.add("%s:%s" % (cpv_getkey(cpv), slot)) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) return rValue class LibraryFileConsumerSet(LibraryConsumerSet): @@ -49,7 +49,8 @@ class LibraryFileConsumerSet(LibraryConsumerSet): def load(self): consumers = set() for lib in self.files: - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update( + self.dbapi._linkmap.findConsumers(lib, greedy=False)) if not consumers: return @@ -77,10 +78,10 @@ class PreservedLibraryConsumerSet(LibraryConsumerSet): for lib in libs: if self.debug: print(lib) - for x in sorted(self.dbapi._linkmap.findConsumers(lib)): + for x in sorted(self.dbapi._linkmap.findConsumers(lib, greedy=False)): print(" ", x) print("-"*40) - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update(self.dbapi._linkmap.findConsumers(lib, greedy=False)) # Don't rebuild packages just because they contain preserved # libs that happen to be consumers of other preserved libs. for libs in plib_dict.values(): diff --git a/portage_with_autodep/pym/portage/_sets/libs.pyo b/portage_with_autodep/pym/portage/_sets/libs.pyo Binary files differindex 72fc1bb..7a9f6d1 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.pyo +++ b/portage_with_autodep/pym/portage/_sets/libs.pyo diff --git a/portage_with_autodep/pym/portage/_sets/profiles.pyo b/portage_with_autodep/pym/portage/_sets/profiles.pyo Binary files differindex 9502044..9ce72f1 100644 --- a/portage_with_autodep/pym/portage/_sets/profiles.pyo +++ b/portage_with_autodep/pym/portage/_sets/profiles.pyo diff --git a/portage_with_autodep/pym/portage/_sets/security.py b/portage_with_autodep/pym/portage/_sets/security.py index 7e856bc..f8dbef2 100644 --- a/portage_with_autodep/pym/portage/_sets/security.py +++ b/portage_with_autodep/pym/portage/_sets/security.py @@ -44,8 +44,8 @@ class SecuritySet(PackageSet): mydict = {} for atom in atomlist[:]: cpv = self._portdbapi.xmatch("match-all", atom)[0] - slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] - cps = "%s:%s" % (cpv.cp, slot) + pkg = self._portdbapi._pkg_str(cpv, None) + cps = "%s:%s" % (pkg.cp, pkg.slot) if not cps in mydict: mydict[cps] = (atom, cpv) else: diff --git a/portage_with_autodep/pym/portage/_sets/security.pyo b/portage_with_autodep/pym/portage/_sets/security.pyo Binary files differindex ea67514..9603bc8 100644 --- a/portage_with_autodep/pym/portage/_sets/security.pyo +++ b/portage_with_autodep/pym/portage/_sets/security.pyo diff --git a/portage_with_autodep/pym/portage/_sets/shell.pyo b/portage_with_autodep/pym/portage/_sets/shell.pyo Binary files differindex e5e4561..f153c51 100644 --- a/portage_with_autodep/pym/portage/_sets/shell.pyo +++ b/portage_with_autodep/pym/portage/_sets/shell.pyo diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differindex eb5a90e..735ab0b 100644 --- a/portage_with_autodep/pym/portage/cache/__init__.pyo +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differindex 5946da9..8422437 100644 --- a/portage_with_autodep/pym/portage/cache/anydbm.pyo +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differindex 866088e..e323de7 100644 --- a/portage_with_autodep/pym/portage/cache/cache_errors.pyo +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 0086e40..db6e177 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,4 +1,4 @@ -# -*- coding: UTF8 -*- +# -*- coding: utf-8 -*- # Copyright: 2009-2011 Gentoo Foundation # Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differindex fe32dcc..405f372 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index 2eae9f6..08dcbe8 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -1,7 +1,9 @@ -# Copyright: 2005-2011 Gentoo Foundation +# Copyright: 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + from portage.cache import fs_template from portage.cache import cache_errors import errno @@ -11,16 +13,13 @@ import sys import os as _os from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode +from portage.exception import InvalidData +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s=%s\n") - class database(fs_template.FsBased): autocommits = True @@ -40,11 +39,10 @@ class database(fs_template.FsBased): # Don't use os.path.join, for better performance. fp = self.location + _os.sep + cpv try: - myf = io.open(_unicode_encode(fp, + with io.open(_unicode_encode(fp, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: + errors='replace') as myf: lines = myf.read().split("\n") if not lines[-1]: lines.pop() @@ -54,8 +52,6 @@ class database(fs_template.FsBased): # that uses mtime mangling. d['_mtime_'] = _os.fstat(myf.fileno())[stat.ST_MTIME] return d - finally: - myf.close() except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise cache_errors.CacheCorruption(cpv, e) @@ -94,7 +90,10 @@ class database(fs_template.FsBased): v = values.get(k) if not v: continue - myf.write(_setitem_fmt % (k, v)) + # NOTE: This format string requires unicode_literals, so that + # k and v are coerced to unicode, in order to prevent TypeError + # when writing raw bytes to TextIOWrapper with Python 2. + myf.write("%s=%s\n" % (k, v)) finally: myf.close() self._ensure_access(fp) @@ -135,8 +134,6 @@ class database(fs_template.FsBased): del e continue for l in dir_list: - if l.endswith(".cpickle"): - continue p = os.path.join(dir_path, l) try: st = os.lstat(p) @@ -151,7 +148,11 @@ class database(fs_template.FsBased): if depth < 1: dirs.append((depth+1, p)) continue - yield p[len_base+1:] + + try: + yield _pkg_str(p[len_base+1:]) + except InvalidData: + continue class md5_database(database): diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differindex 4f568a8..90a1b36 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.pyo +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.py b/portage_with_autodep/pym/portage/cache/flat_list.py deleted file mode 100644 index 7288307..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2005-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import fs_template -from portage.cache import cache_errors -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -import errno -import io -import stat -import sys - -if sys.hexversion >= 0x3000000: - long = int - -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s\n") - -# store the current key order *here*. -class database(fs_template.FsBased): - - autocommits = True - - # do not screw with this ordering. _eclasses_ needs to be last - auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', - 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', - 'KEYWORDS', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') - - def __init__(self, *args, **config): - super(database,self).__init__(*args, **config) - self.location = os.path.join(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) - - if len(self._known_keys) > len(self.auxdbkey_order) + 2: - raise Exception("less ordered keys then auxdbkeys") - if not os.path.exists(self.location): - self._ensure_dirs() - - - def _getitem(self, cpv): - d = {} - try: - myf = io.open(_unicode_encode(os.path.join(self.location, cpv), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - for k,v in zip(self.auxdbkey_order, myf): - d[k] = v.rstrip("\n") - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - raise cache_errors.CacheCorruption(cpv, e) - - try: - d["_mtime_"] = os.fstat(myf.fileno())[stat.ST_MTIME] - except OSError as e: - myf.close() - raise cache_errors.CacheCorruption(cpv, e) - myf.close() - return d - - - def _setitem(self, cpv, values): - s = cpv.rfind("/") - fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) - try: - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - try: - self._ensure_dirs(cpv) - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - raise cache_errors.CacheCorruption(cpv, e) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - for x in self.auxdbkey_order: - myf.write(_setitem_fmt % (values.get(x, ""),)) - - myf.close() - self._ensure_access(fp, mtime=values["_mtime_"]) - #update written. now we move it. - new_fp = os.path.join(self.location,cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError) as e: - os.remove(fp) - raise cache_errors.CacheCorruption(cpv, e) - - - def _delitem(self, cpv): - try: - os.remove(os.path.join(self.location,cpv)) - except OSError as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - def __contains__(self, cpv): - return os.path.exists(os.path.join(self.location, cpv)) - - - def __iter__(self): - """generator for walking the dir struct""" - dirs = [self.location] - len_base = len(self.location) - while len(dirs): - for l in os.listdir(dirs[0]): - if l.endswith(".cpickle"): - continue - p = os.path.join(dirs[0],l) - st = os.lstat(p) - if stat.S_ISDIR(st.st_mode): - dirs.append(p) - continue - yield p[len_base+1:] - dirs.pop(0) - - - def commit(self): pass diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differdeleted file mode 100644 index ab7dc82..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/cache/fs_template.py b/portage_with_autodep/pym/portage/cache/fs_template.py index a82e862..0567c72 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.py +++ b/portage_with_autodep/pym/portage/cache/fs_template.py @@ -1,14 +1,14 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 +import os as _os import sys from portage.cache import template from portage import os from portage.proxy.lazyimport import lazyimport lazyimport(globals(), - 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) @@ -22,14 +22,11 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - """throws InitializationError if needs args aren't specified - gid and perms aren't listed do to an oddity python currying mechanism - gid=portage_gid - perms=0665""" for x, y in (("gid", -1), ("perms", -1)): if x in config: - setattr(self, "_"+x, config[x]) + # Since Python 3.4, chown requires int type (no proxies). + setattr(self, "_" + x, int(config[x])) del config[x] else: setattr(self, "_"+x, y) @@ -78,7 +75,17 @@ class FsBased(template.database): if self._perms != -1: os.umask(um) - + def _prune_empty_dirs(self): + all_dirs = [] + for parent, dirs, files in os.walk(self.location): + for x in dirs: + all_dirs.append(_os.path.join(parent, x)) + while all_dirs: + try: + _os.rmdir(all_dirs.pop()) + except OSError: + pass + def gen_label(base, label): """if supplied label is a path, generate a unique label based upon label, and supplied base path""" if label.find(os.path.sep) == -1: diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differindex 6cbbc2f..de71ee6 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.pyo +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index bc8ce9a..cd39a6e 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -199,10 +199,10 @@ class OrderedDict(UserDict): return iter(self._order) def __setitem__(self, key, item): - if key in self: - self._order.remove(key) + new_key = key not in self UserDict.__setitem__(self, key, item) - self._order.append(key) + if new_key: + self._order.append(key) def __delitem__(self, key): UserDict.__delitem__(self, key) diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differindex 1eb3f4f..c183246 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.pyo +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 9d2c3a5..6612e73 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -28,7 +28,8 @@ class database(flat_hash.database): auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') + 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', + 'DEFINED_PHASES', 'HDEPEND') autocommits = True serialize_eclasses = False diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differindex c98445b..649ec5b 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.pyo +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differindex e2c5974..54636ed 100644 --- a/portage_with_autodep/pym/portage/cache/sql_template.pyo +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.py b/portage_with_autodep/pym/portage/cache/sqlite.py index fcc62ff..40db070 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.py +++ b/portage_with_autodep/pym/portage/cache/sqlite.py @@ -1,6 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import re import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -20,7 +23,6 @@ class database(fs_template.FsBased): # to calculate the number of pages requested, according to the following # equation: cache_bytes = page_bytes * page_count cache_bytes = 1024 * 1024 * 10 - _db_table = None def __init__(self, *args, **config): super(database, self).__init__(*args, **config) @@ -28,6 +30,7 @@ class database(fs_template.FsBased): self._allowed_keys = ["_mtime_", "_eclasses_"] self._allowed_keys.extend(self._known_keys) self._allowed_keys.sort() + self._allowed_keys_set = frozenset(self._allowed_keys) self.location = os.path.join(self.location, self.label.lstrip(os.path.sep).rstrip(os.path.sep)) @@ -37,8 +40,8 @@ class database(fs_template.FsBased): config.setdefault("autocommit", self.autocommits) config.setdefault("cache_bytes", self.cache_bytes) config.setdefault("synchronous", self.synchronous) - # Timeout for throwing a "database is locked" exception (pysqlite - # default is 5.0 seconds). + # Set longer timeout for throwing a "database is locked" exception. + # Default timeout in sqlite3 module is 5.0 seconds. config.setdefault("timeout", 15) self._db_init_connection(config) self._db_init_structures() @@ -47,11 +50,8 @@ class database(fs_template.FsBased): # sqlite3 is optional with >=python-2.5 try: import sqlite3 as db_module - except ImportError: - try: - from pysqlite2 import dbapi2 as db_module - except ImportError as e: - raise cache_errors.InitializationError(self.__class__, e) + except ImportError as e: + raise cache_errors.InitializationError(self.__class__, e) self._db_module = db_module self._db_error = db_module.Error @@ -62,7 +62,6 @@ class database(fs_template.FsBased): # Avoid potential UnicodeEncodeError in python-2.x by # only calling str() when it's absolutely necessary. s = str(s) - # This is equivalent to the _quote function from pysqlite 1.1. return "'%s'" % s.replace("'", "''") def _db_init_connection(self, config): @@ -92,9 +91,6 @@ class database(fs_template.FsBased): self._db_table["packages"]["table_name"] = mytable self._db_table["packages"]["package_id"] = "internal_db_package_id" self._db_table["packages"]["package_key"] = "portage_package_key" - self._db_table["packages"]["internal_columns"] = \ - [self._db_table["packages"]["package_id"], - self._db_table["packages"]["package_key"]] create_statement = [] create_statement.append("CREATE TABLE") create_statement.append(mytable) @@ -109,15 +105,18 @@ class database(fs_template.FsBased): create_statement.append(")") self._db_table["packages"]["create"] = " ".join(create_statement) - self._db_table["packages"]["columns"] = \ - self._db_table["packages"]["internal_columns"] + \ - self._allowed_keys cursor = self._db_cursor for k, v in self._db_table.items(): if self._db_table_exists(v["table_name"]): create_statement = self._db_table_get_create(v["table_name"]) - if create_statement != v["create"]: + table_ok, missing_keys = self._db_validate_create_statement(create_statement) + if table_ok: + if missing_keys: + for k in sorted(missing_keys): + cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % + (self._db_table["packages"]["table_name"], k)) + else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) cursor.execute("DROP TABLE %s" % v["table_name"]) cursor.execute(v["create"]) @@ -138,6 +137,37 @@ class database(fs_template.FsBased): self._db_escape_string(table_name)) return cursor.fetchall()[0][0] + def _db_validate_create_statement(self, statement): + missing_keys = None + if statement == self._db_table["packages"]["create"]: + return True, missing_keys + + m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % + (self._db_table["packages"]["table_name"], + self._db_table["packages"]["package_id"]), + statement) + if m is None: + return False, missing_keys + + unique_constraints = set([self._db_table["packages"]["package_key"]]) + missing_keys = set(self._allowed_keys) + unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') + column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') + for x in m.group(1).split(","): + m = column_re.match(x) + if m is not None: + missing_keys.discard(m.group(1)) + continue + m = unique_re.match(x) + if m is not None: + unique_constraints.discard(m.group(1)) + continue + + if unique_constraints: + return False, missing_keys + + return True, missing_keys + def _db_init_cache_size(self, cache_bytes): cursor = self._db_cursor cursor.execute("PRAGMA page_size") @@ -173,13 +203,17 @@ class database(fs_template.FsBased): raise KeyError(cpv) else: raise cache_errors.CacheCorruption(cpv, "key is not unique") + result = result[0] d = {} - internal_columns = self._db_table["packages"]["internal_columns"] - column_index = -1 - for k in self._db_table["packages"]["columns"]: - column_index +=1 - if k not in internal_columns: - d[k] = result[0][column_index] + allowed_keys_set = self._allowed_keys_set + for column_index, column_info in enumerate(cursor.description): + k = column_info[0] + if k in allowed_keys_set: + v = result[column_index] + if v is None: + # This happens after a new empty column has been added. + v = "" + d[k] = v return d diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differindex a82d25f..aba2f6c 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.pyo +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index cf1e8ae..9b8a4d3 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,6 +1,6 @@ -# Copyright: 2005-2012 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 from portage.cache import cache_errors from portage.cache.cache_errors import InvalidRestriction @@ -164,7 +164,14 @@ class database(object): def commit(self): if not self.autocommits: - raise NotImplementedError + raise NotImplementedError(self) + + def __del__(self): + # This used to be handled by an atexit hook that called + # close_portdbapi_caches() for all portdbapi instances, but that was + # prone to memory leaks for API consumers that needed to create/destroy + # many portdbapi instances. So, instead we rely on __del__. + self.sync() def __contains__(self, cpv): """This method should always be overridden. It is provided only for diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differindex 45da015..3315771 100644 --- a/portage_with_autodep/pym/portage/cache/template.pyo +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differindex fac5d55..9ebb5b8 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.pyo +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo diff --git a/portage_with_autodep/pym/portage/checksum.py b/portage_with_autodep/pym/portage/checksum.py index bd416ac..cd1572e 100644 --- a/portage_with_autodep/pym/portage/checksum.py +++ b/portage_with_autodep/pym/portage/checksum.py @@ -1,5 +1,5 @@ # checksum.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -10,6 +10,8 @@ from portage import _encodings from portage import _unicode_encode import errno import stat +import sys +import subprocess import tempfile #dict of all available hash functions @@ -48,16 +50,15 @@ class _generate_hash_function(object): @type filename: String @return: The hash and size of the data """ - f = _open_file(filename) - blocksize = HASHING_BLOCKSIZE - data = f.read(blocksize) - size = 0 - checksum = self._hashobject() - while data: - checksum.update(data) - size = size + len(data) + with _open_file(filename) as f: + blocksize = HASHING_BLOCKSIZE + size = 0 + checksum = self._hashobject() data = f.read(blocksize) - f.close() + while data: + checksum.update(data) + size = size + len(data) + data = f.read(blocksize) return (checksum.hexdigest(), size) @@ -137,8 +138,10 @@ try: except ImportError: pass +_whirlpool_unaccelerated = False if "WHIRLPOOL" not in hashfunc_map: # Bundled WHIRLPOOL implementation + _whirlpool_unaccelerated = True from portage.util.whirlpool import new as _new_whirlpool whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") @@ -161,11 +164,16 @@ hashfunc_map["size"] = getsize prelink_capable = False if os.path.exists(PRELINK_BINARY): - results = portage.subprocess_getstatusoutput( - "%s --version > /dev/null 2>&1" % (PRELINK_BINARY,)) - if (results[0] >> 8) == 0: + cmd = [PRELINK_BINARY, "--version"] + cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + proc.communicate() + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: prelink_capable=1 - del results + del cmd, proc, status def is_prelinkable_elf(filename): f = _open_file(filename) @@ -197,6 +205,82 @@ def get_hash_origin(hashtype): raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") +def _filter_unaccelarated_hashes(digests): + """ + If multiple digests are available and some are unaccelerated, + then return a new dict that omits the unaccelerated ones. This + allows extreme performance problems like bug #425046 to be + avoided whenever practical, especially for cases like stage + builds where acceleration may not be available for some hashes + due to minimization of dependencies. + """ + if _whirlpool_unaccelerated and "WHIRLPOOL" in digests: + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + if len(verifiable_hash_types) > 1: + digests = dict(digests) + digests.pop("WHIRLPOOL") + + return digests + +class _hash_filter(object): + """ + Implements filtering for PORTAGE_CHECKSUM_FILTER. + """ + + __slots__ = ('transparent', '_tokens',) + + def __init__(self, filter_str): + tokens = filter_str.upper().split() + if not tokens or tokens[-1] == "*": + del tokens[:] + self.transparent = not tokens + tokens.reverse() + self._tokens = tuple(tokens) + + def __call__(self, hash_name): + if self.transparent: + return True + matches = ("*", hash_name) + for token in self._tokens: + if token in matches: + return True + elif token[:1] == "-": + if token[1:] in matches: + return False + return False + +def _apply_hash_filter(digests, hash_filter): + """ + Return a new dict containing the filtered digests, or the same + dict if no changes are necessary. This will always preserve at + at least one digest, in order to ensure that they are not all + discarded. + @param digests: dictionary of digests + @type digests: dict + @param hash_filter: A callable that takes a single hash name + argument, and returns True if the hash is to be used or + False otherwise + @type hash_filter: callable + """ + + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + modified = False + if len(verifiable_hash_types) > 1: + for k in list(verifiable_hash_types): + if not hash_filter(k): + modified = True + verifiable_hash_types.remove(k) + if len(verifiable_hash_types) == 1: + break + + if modified: + digests = dict((k, v) for (k, v) in digests.items() + if k == "size" or k in verifiable_hash_types) + + return digests + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. diff --git a/portage_with_autodep/pym/portage/checksum.pyo b/portage_with_autodep/pym/portage/checksum.pyo Binary files differindex 00231af..46441a3 100644 --- a/portage_with_autodep/pym/portage/checksum.pyo +++ b/portage_with_autodep/pym/portage/checksum.pyo diff --git a/portage_with_autodep/pym/portage/const.py b/portage_with_autodep/pym/portage/const.py index 614dcdb..0ac503f 100644 --- a/portage_with_autodep/pym/portage/const.py +++ b/portage_with_autodep/pym/portage/const.py @@ -1,7 +1,9 @@ # portage: Constants -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import os # =========================================================================== @@ -27,8 +29,8 @@ import os # The variables in this file are grouped by config_root, target_root. # variables used with config_root (these need to be relative) -MAKE_CONF_FILE = "etc/make.conf" USER_CONFIG_PATH = "etc/portage" +MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf" MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules" CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile" USER_VIRTUALS_FILE = USER_CONFIG_PATH + "/virtuals" @@ -36,7 +38,7 @@ EBUILD_SH_ENV_FILE = USER_CONFIG_PATH + "/bashrc" EBUILD_SH_ENV_DIR = USER_CONFIG_PATH + "/env" CUSTOM_MIRRORS_FILE = USER_CONFIG_PATH + "/mirrors" COLOR_MAP_FILE = USER_CONFIG_PATH + "/color.map" -PROFILE_PATH = "etc/make.profile" +PROFILE_PATH = USER_CONFIG_PATH + "/make.profile" MAKE_DEFAULTS_FILE = PROFILE_PATH + "/make.defaults" # FIXME: not used DEPRECATED_PROFILE_FILE = PROFILE_PATH + "/deprecated" @@ -56,7 +58,10 @@ DEPCACHE_PATH = "/var/cache/edb/dep" GLOBAL_CONFIG_PATH = "/usr/share/portage/config" # these variables are not used with target_root or config_root -PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(__file__.split(os.sep)[:-3])) +# NOTE: Use realpath(__file__) so that python module symlinks in site-packages +# are followed back to the real location of the whole portage installation. +PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath( + __file__.rstrip("co")).split(os.sep)[:-3])) PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin" PORTAGE_PYM_PATH = PORTAGE_BASE_PATH + "/pym" LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale" # FIXME: not used @@ -69,7 +74,6 @@ MOVE_BINARY = "/bin/mv" PRELINK_BINARY = "/usr/sbin/prelink" AUTODEP_LIBRARY = "/usr/lib/file_hook.so" - INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" REPO_NAME_FILE = "repo_name" REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE @@ -77,50 +81,139 @@ REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE PORTAGE_PACKAGE_ATOM = "sys-apps/portage" LIBC_PACKAGE_ATOM = "virtual/libc" OS_HEADERS_PACKAGE_ATOM = "virtual/os-headers" +CVS_PACKAGE_ATOM = "dev-vcs/cvs" +GIT_PACKAGE_ATOM = "dev-vcs/git" +RSYNC_PACKAGE_ATOM = "net-misc/rsync" -INCREMENTALS = ("USE", "USE_EXPAND", "USE_EXPAND_HIDDEN", - "FEATURES", "ACCEPT_KEYWORDS", - "CONFIG_PROTECT_MASK", "CONFIG_PROTECT", - "PRELINK_PATH", "PRELINK_PATH_MASK", - "PROFILE_ONLY_VARIABLES") -EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure", - "compile", "test", "install", - "package", "preinst", "postinst","prerm", "postrm", - "nofetch", "config", "info", "other") +INCREMENTALS = ( + "ACCEPT_KEYWORDS", + "CONFIG_PROTECT", + "CONFIG_PROTECT_MASK", + "FEATURES", + "IUSE_IMPLICIT", + "PRELINK_PATH", + "PRELINK_PATH_MASK", + "PROFILE_ONLY_VARIABLES", + "USE", + "USE_EXPAND", + "USE_EXPAND_HIDDEN", + "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED", +) +EBUILD_PHASES = ( + "pretend", + "setup", + "unpack", + "prepare", + "configure", + "compile", + "test", + "install", + "package", + "preinst", + "postinst", + "prerm", + "postrm", + "nofetch", + "config", + "info", + "other", +) SUPPORTED_FEATURES = frozenset([ - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", - "ccache", "chflags", "clean-logs", - "collision-protect", "compress-build-logs", "compressdebug", - "config-protect-if-modified", "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "force-mirror", "force-prefix", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", - "metadata-transfer", "mirror", "multilib-strict", "news", - "noauto", "noclean", "nodoc", "noinfo", "noman", - "nostrip", "notitles", "parallel-fetch", "parallel-install", - "parse-eapi-ebuild-head", - "prelink-checksums", "preserve-libs", - "protect-owned", "python-trace", "sandbox", - "selinux", "sesandbox", "sfperms", - "sign", "skiprocheck", "split-elog", "split-log", "splitdebug", - "strict", "stricter", "suidctl", "test", "test-fail-continue", - "unknown-features-filter", "unknown-features-warn", - "unmerge-logs", "unmerge-orphans", "userfetch", "userpriv", - "usersandbox", "usersync", "webrsync-gpg", "xattr"]) - -EAPI = 4 + "assume-digests", + "binpkg-logs", + "buildpkg", + "buildsyspkg", + "candy", + "ccache", + "cgroup", + "chflags", + "clean-logs", + "collision-protect", + "compress-build-logs", + "compressdebug", + "compress-index", + "config-protect-if-modified", + "depcheck", + "depcheckstrict", + "digest", + "distcc", + "distcc-pump", + "distlocks", + "downgrade-backup", + "ebuild-locks", + "fail-clean", + "fakeroot", + "fixlafiles", + "force-mirror", + "force-prefix", + "getbinpkg", + "installsources", + "ipc-sandbox", + "keeptemp", + "keepwork", + "lmirror", + "merge-sync", + "metadata-transfer", + "mirror", + "multilib-strict", + "network-sandbox", + "news", + "noauto", + "noclean", + "nodoc", + "noinfo", + "noman", + "nostrip", + "notitles", + "parallel-fetch", + "parallel-install", + "prelink-checksums", + "preserve-libs", + "protect-owned", + "python-trace", + "sandbox", + "selinux", + "sesandbox", + "sfperms", + "sign", + "skiprocheck", + "splitdebug", + "split-elog", + "split-log", + "strict", + "stricter", + "suidctl", + "test", + "test-fail-continue", + "unknown-features-filter", + "unknown-features-warn", + "unmerge-backup", + "unmerge-logs", + "unmerge-orphans", + "userfetch", + "userpriv", + "usersandbox", + "usersync", + "webrsync-gpg", + "xattr", +]) + +EAPI = 5 HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ("MD5", "SHA256", "RMD160") MANIFEST1_REQUIRED_HASH = "MD5" -# Future events: +# Past events: # -# After WHIRLPOOL is supported in stable portage: -# - Add SHA256 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. -# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# 20120704 - After WHIRLPOOL is supported in stable portage: # - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: # manifest-hashes = SHA256 SHA512 WHIRLPOOL +# - Add SHA512 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. +# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# +# Future events: # # After WHIRLPOOL is supported in stable portage for at least 1 year: # - Change MANIFEST2_REQUIRED_HASH to WHIRLPOOL. @@ -138,8 +231,8 @@ MANIFEST1_REQUIRED_HASH = "MD5" # After layout.conf settings correspond to defaults in stable portage: # - Remove redundant settings from gentoo-x86/metadata/layout.conf. -MANIFEST2_HASH_FUNCTIONS = ("RMD160", "SHA1", "SHA256", "SHA512", "WHIRLPOOL") -MANIFEST2_HASH_DEFAULTS = frozenset(["SHA1", "SHA256", "RMD160"]) +MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL") +MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"]) MANIFEST2_REQUIRED_HASH = "SHA256" MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") @@ -149,13 +242,35 @@ MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") # a config instance (since it's possible to contruct a config instance with # a different EPREFIX). Therefore, the EPREFIX constant should *NOT* be used # in the definition of any other constants within this file. -EPREFIX="" +EPREFIX = "" # pick up EPREFIX from the environment if set if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: EPREFIX = os.environ["PORTAGE_OVERRIDE_EPREFIX"] if EPREFIX: EPREFIX = os.path.normpath(EPREFIX) + if EPREFIX == os.sep: + EPREFIX = "" + +VCS_DIRS = ("CVS", "RCS", "SCCS", ".bzr", ".git", ".hg", ".svn") + +# List of known live eclasses. Keep it in sync with cnf/sets/portage.conf +LIVE_ECLASSES = frozenset([ + "bzr", + "cvs", + "darcs", + "git", + "git-2", + "git-r3", + "mercurial", + "subversion", + "tla", +]) + +SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") + +# Time formats used in various places like metadata.chk. +TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT @@ -163,19 +278,5 @@ if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: # Private constants for use in conditional code in order to minimize the diff # between branches. -_ENABLE_DYN_LINK_MAP = True -_ENABLE_PRESERVE_LIBS = True -_ENABLE_REPO_NAME_WARN = True +_DEPCLEAN_LIB_CHECK_DEFAULT = True _ENABLE_SET_CONFIG = True - - -# The definitions above will differ between branches, so it's useful to have -# common lines of diff context here in order to avoid merge conflicts. - -if not _ENABLE_PRESERVE_LIBS: - SUPPORTED_FEATURES = set(SUPPORTED_FEATURES) - SUPPORTED_FEATURES.remove("preserve-libs") - SUPPORTED_FEATURES = frozenset(SUPPORTED_FEATURES) - -if not _ENABLE_SET_CONFIG: - WORLD_SETS_FILE = '/dev/null' diff --git a/portage_with_autodep/pym/portage/const.py.rej b/portage_with_autodep/pym/portage/const.py.rej deleted file mode 100644 index 9fe70f8..0000000 --- a/portage_with_autodep/pym/portage/const.py.rej +++ /dev/null @@ -1,12 +0,0 @@ ---- pym/portage/const.py -+++ pym/portage/const.py -@@ -90,7 +92,8 @@ - SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", -- "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "fixpackages", "force-mirror", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", diff --git a/portage_with_autodep/pym/portage/const.pyo b/portage_with_autodep/pym/portage/const.pyo Binary files differindex 804420f..3b9812b 100644 --- a/portage_with_autodep/pym/portage/const.pyo +++ b/portage_with_autodep/pym/portage/const.pyo diff --git a/portage_with_autodep/pym/portage/cvstree.pyo b/portage_with_autodep/pym/portage/cvstree.pyo Binary files differindex 4719daf..1c18d23 100644 --- a/portage_with_autodep/pym/portage/cvstree.pyo +++ b/portage_with_autodep/pym/portage/cvstree.pyo diff --git a/portage_with_autodep/pym/portage/data.py b/portage_with_autodep/pym/portage/data.py index c4d967a..44104c2 100644 --- a/portage_with_autodep/pym/portage/data.py +++ b/portage_with_autodep/pym/portage/data.py @@ -1,13 +1,14 @@ # data.py -- Calculated/Discovered Data Values -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import os, pwd, grp, platform +import os, pwd, grp, platform, sys import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.output:colorize', 'portage.util:writemsg', + 'subprocess' ) from portage.localization import _ @@ -85,19 +86,27 @@ def _get_global(k): elif portage.const.EPREFIX: secpass = 2 #Discover the uid and gid of the portage user/group + keyerror = False try: portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid - _portage_grpname = _get_global('_portage_grpname') - if platform.python_implementation() == 'PyPy': - # Somehow this prevents "TypeError: expected string" errors - # from grp.getgrnam() with PyPy 1.7 - _portage_grpname = str(_portage_grpname) - portage_gid = grp.getgrnam(_portage_grpname).gr_gid - if secpass < 1 and portage_gid in os.getgroups(): - secpass = 1 except KeyError: + keyerror = True portage_uid = 0 + + try: + portage_gid = grp.getgrnam(_get_global('_portage_grpname')).gr_gid + except KeyError: + keyerror = True portage_gid = 0 + + if secpass < 1 and portage_gid in os.getgroups(): + secpass = 1 + + # Suppress this error message if both PORTAGE_GRPNAME and + # PORTAGE_USERNAME are set to "root", for things like + # Android (see bug #454060). + if keyerror and not (_get_global('_portage_username') == "root" and + _get_global('_portage_grpname') == "root"): writemsg(colorize("BAD", _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) writemsg(_( @@ -129,10 +138,28 @@ def _get_global(k): # Get a list of group IDs for the portage user. Do not use # grp.getgrall() since it is known to trigger spurious # SIGPIPE problems with nss_ldap. - mystatus, myoutput = \ - portage.subprocess_getstatusoutput("id -G %s" % _portage_username) - if mystatus == os.EX_OK: - for x in myoutput.split(): + cmd = ["id", "-G", _portage_username] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + globals()[k] = v + _initialized_globals.add(k) + return v + cmd[0] = fullname + + encoding = portage._encodings['content'] + cmd = [portage._unicode_encode(x, + encoding=encoding, errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + myoutput = proc.communicate()[0] + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: + for x in portage._unicode_decode(myoutput, + encoding=encoding, errors='strict').split(): try: v.append(int(x)) except ValueError: @@ -213,10 +240,18 @@ def _init(settings): if '_portage_grpname' not in _initialized_globals and \ '_portage_username' not in _initialized_globals: + # Prevents "TypeError: expected string" errors + # from grp.getgrnam() with PyPy + native_string = platform.python_implementation() == 'PyPy' + v = settings.get('PORTAGE_GRPNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_grpname'] = v _initialized_globals.add('_portage_grpname') v = settings.get('PORTAGE_USERNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_username'] = v _initialized_globals.add('_portage_username') diff --git a/portage_with_autodep/pym/portage/data.pyo b/portage_with_autodep/pym/portage/data.pyo Binary files differindex 7f749e0..8115c7a 100644 --- a/portage_with_autodep/pym/portage/data.pyo +++ b/portage_with_autodep/pym/portage/data.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py index b5f6a0b..956dbb9 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py @@ -1,7 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import signal import sys import traceback @@ -10,10 +11,11 @@ import errno import fcntl import portage from portage import os, _unicode_decode +from portage.util._ctypes import find_library import portage.elog.messages -from _emerge.SpawnProcess import SpawnProcess +from portage.util._async.ForkProcess import ForkProcess -class MergeProcess(SpawnProcess): +class MergeProcess(ForkProcess): """ Merge packages in a subprocess, so the Scheduler can run in the main thread while files are moved or copied asynchronously. @@ -40,11 +42,20 @@ class MergeProcess(SpawnProcess): settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if platform.system() == "Linux" and \ + "merge-sync" in settings.features: + find_library("c") + # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} - self.fd_pipes.setdefault(0, sys.stdin.fileno()) + else: + self.fd_pipes = self.fd_pipes.copy() + self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start() @@ -90,7 +101,7 @@ class MergeProcess(SpawnProcess): reporter(msg, phase=phase, key=key, out=out) if event & self.scheduler.IO_HUP: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None os.close(self._elog_reader_fd) self._elog_reader_fd = None @@ -101,12 +112,24 @@ class MergeProcess(SpawnProcess): def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call - dblink.merge(). + dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFD, + fcntl.fcntl(elog_reader_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -116,10 +139,9 @@ class MergeProcess(SpawnProcess): blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, - blockers=blockers, scheduler=self.scheduler, - pipe=elog_writer_fd) + blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd - self._elog_reg_id = self.scheduler.register(elog_reader_fd, + self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package @@ -133,88 +155,100 @@ class MergeProcess(SpawnProcess): if not self.unmerge: counter = self.vartree.dbapi.counter_tick() - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - portage.process.spawned_pids.append(pid) - return [pid] - - os.close(elog_reader_fd) - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers since the ones inherited - # from the parent process are irrelevant here. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - portage.output.havecolor = self.settings.get('NOCOLOR') \ - not in ('yes', 'true') - - # In this subprocess we want mylink._display_merge() to use - # stdout/stderr directly since they are pipes. This behavior - # is triggered when mylink._scheduler is None. - mylink._scheduler = None - - # Avoid wastful updates of the vdb cache. - self.vartree.dbapi._flush_cache_enabled = False - - # In this subprocess we don't want PORTAGE_BACKGROUND to - # suppress stdout/stderr output since they are pipes. We - # also don't want to open PORTAGE_LOG_FILE, since it will - # already be opened by the parent process, so we set the - # "subprocess" value for use in conditional logging code - # involving PORTAGE_LOG_FILE. - if not self.unmerge: - # unmerge phases have separate logs - if self.settings.get("PORTAGE_BACKGROUND") == "1": - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" - else: - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" - self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") - self.settings["PORTAGE_BACKGROUND"] = "subprocess" - self.settings.backup_changes("PORTAGE_BACKGROUND") - - rval = 1 + parent_pid = os.getpid() + pid = None try: - if self.unmerge: - if not mylink.exists(): - rval = os.EX_OK - elif mylink.unmerge( - ldpath_mtimes=self.prev_mtimes) == os.EX_OK: - mylink.lockdb() - try: - mylink.delete() - finally: - mylink.unlockdb() - rval = os.EX_OK - else: - rval = mylink.merge(self.pkgloc, self.infloc, - myebuild=self.myebuild, mydbapi=self.mydbapi, - prev_mtimes=self.prev_mtimes, counter=counter) - except SystemExit: - raise - except: - traceback.print_exc() + pid = os.fork() + + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + + os.close(elog_writer_fd) + self._elog_reader_fd = elog_reader_fd + self._buf = "" + self._elog_keys = set() + # Discard messages which will be collected by the subprocess, + # in order to avoid duplicates (bug #446136). + portage.elog.messages.collect_messages(key=mylink.mycpv) + + # invalidate relevant vardbapi caches + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None + self.vartree.dbapi._pkgs_changed = True + self.vartree.dbapi._clear_pkg_cache(mylink) + + return [pid] + + os.close(elog_reader_fd) + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + portage.locks._close_fds() + # We don't exec, so use close_fds=False + # (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + portage.output.havecolor = self.settings.get('NOCOLOR') \ + not in ('yes', 'true') + + # Avoid wastful updates of the vdb cache. + self.vartree.dbapi._flush_cache_enabled = False + + # In this subprocess we don't want PORTAGE_BACKGROUND to + # suppress stdout/stderr output since they are pipes. We + # also don't want to open PORTAGE_LOG_FILE, since it will + # already be opened by the parent process, so we set the + # "subprocess" value for use in conditional logging code + # involving PORTAGE_LOG_FILE. + if not self.unmerge: + # unmerge phases have separate logs + if self.settings.get("PORTAGE_BACKGROUND") == "1": + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" + else: + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" + self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") + self.settings["PORTAGE_BACKGROUND"] = "subprocess" + self.settings.backup_changes("PORTAGE_BACKGROUND") + + rval = 1 + try: + if self.unmerge: + if not mylink.exists(): + rval = os.EX_OK + elif mylink.unmerge( + ldpath_mtimes=self.prev_mtimes) == os.EX_OK: + mylink.lockdb() + try: + mylink.delete() + finally: + mylink.unlockdb() + rval = os.EX_OK + else: + rval = mylink.merge(self.pkgloc, self.infloc, + myebuild=self.myebuild, mydbapi=self.mydbapi, + prev_mtimes=self.prev_mtimes, counter=counter) + except SystemExit: + raise + except: + traceback.print_exc() + # os._exit() skips stderr flush! + sys.stderr.flush() + finally: + os._exit(rval) + finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. + os._exit(1) def _unregister(self): """ @@ -231,7 +265,7 @@ class MergeProcess(SpawnProcess): self._unlock_vdb() if self._elog_reg_id is not None: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None if self._elog_reader_fd is not None: os.close(self._elog_reader_fd) diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo Binary files differindex 5839ad8..abee4be 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.py b/portage_with_autodep/pym/portage/dbapi/__init__.py index a1c5c56..a20a1e8 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.py +++ b/portage_with_autodep/pym/portage/dbapi/__init__.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dbapi"] import re @@ -8,7 +10,7 @@ import re import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand@_dep_expand', - 'portage.dep:match_from_list', + 'portage.dep:Atom,match_from_list,_match_slot', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', @@ -16,14 +18,19 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import os from portage import auxdbkeys +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData from portage.localization import _ +from _emerge.Package import Package class dbapi(object): - _category_re = re.compile(r'^\w[-.+\w]*$') + _category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE) _categories = None _use_mutable = False _known_keys = frozenset(x for x in auxdbkeys if not x.startswith("UNUSED_0")) + _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository") + def __init__(self): pass @@ -125,29 +132,52 @@ class dbapi(object): def _iter_match(self, atom, cpv_iter): cpv_iter = iter(match_from_list(atom, cpv_iter)) + if atom.repo: + cpv_iter = self._iter_match_repo(atom, cpv_iter) if atom.slot: cpv_iter = self._iter_match_slot(atom, cpv_iter) if atom.unevaluated_atom.use: cpv_iter = self._iter_match_use(atom, cpv_iter) - if atom.repo: - cpv_iter = self._iter_match_repo(atom, cpv_iter) return cpv_iter + def _pkg_str(self, cpv, repo): + """ + This is used to contruct _pkg_str instances on-demand during + matching. If cpv is a _pkg_str instance with slot attribute, + then simply return it. Otherwise, fetch metadata and construct + a _pkg_str instance. This may raise KeyError or InvalidData. + """ + try: + cpv.slot + except AttributeError: + pass + else: + return cpv + + metadata = dict(zip(self._pkg_str_aux_keys, + self.aux_get(cpv, self._pkg_str_aux_keys, myrepo=repo))) + + return _pkg_str(cpv, metadata=metadata, settings=self.settings) + def _iter_match_repo(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if pkg_str.repo == atom.repo: + yield pkg_str def _iter_match_slot(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if _match_slot(atom, pkg_str): + yield pkg_str def _iter_match_use(self, atom, cpv_iter): """ @@ -155,7 +185,7 @@ class dbapi(object): 2) Check enabled/disabled flag states. """ - aux_keys = ["IUSE", "SLOT", "USE", "repository"] + aux_keys = ["EAPI", "IUSE", "KEYWORDS", "SLOT", "USE", "repository"] for cpv in cpv_iter: try: metadata = dict(zip(aux_keys, @@ -163,17 +193,31 @@ class dbapi(object): except KeyError: continue + try: + cpv.slot + except AttributeError: + try: + cpv = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + if not self._match_use(atom, cpv, metadata): continue yield cpv - def _match_use(self, atom, cpv, metadata): - iuse_implicit_match = self.settings._iuse_implicit_match - iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split()) + def _match_use(self, atom, pkg, metadata): + eapi_attrs = _get_eapi_attrs(metadata["EAPI"]) + if eapi_attrs.iuse_effective: + iuse_implicit_match = self.settings._iuse_effective_match + else: + iuse_implicit_match = self.settings._iuse_implicit_match + usealiases = self.settings._use_manager.getUseAliases(pkg) + iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"]) for x in atom.unevaluated_atom.use.required: - if x not in iuse and not iuse_implicit_match(x): + if iuse.get_real_flag(x) is None: return False if atom.use is None: @@ -183,45 +227,54 @@ class dbapi(object): # Use IUSE to validate USE settings for built packages, # in case the package manager that built this package # failed to do that for some reason (or in case of - # data corruption). - use = frozenset(x for x in metadata["USE"].split() - if x in iuse or iuse_implicit_match(x)) - missing_enabled = atom.use.missing_enabled.difference(iuse) - missing_disabled = atom.use.missing_disabled.difference(iuse) - - if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + # data corruption). The enabled flags must be consistent + # with implicit IUSE, in order to avoid potential + # inconsistencies in USE dep matching (see bug #453400). + use = frozenset(x for x in metadata["USE"].split() if iuse.get_real_flag(x) is not None) + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + enabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.enabled) + disabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.disabled) + + if enabled: + if any(x in enabled for x in missing_disabled): return False - need_enabled = atom.use.enabled.difference(use) + need_enabled = enabled.difference(use) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(x not in missing_enabled for x in need_enabled): return False - if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if disabled: + if any(x in disabled for x in missing_enabled): return False - need_disabled = atom.use.disabled.intersection(use) + need_disabled = disabled.intersection(use) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(x not in missing_disabled for x in need_disabled): return False elif not self.settings.local_config: # Check masked and forced flags for repoman. - if hasattr(cpv, 'slot'): - pkg = cpv - else: - pkg = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) - usemask = self.settings._getUseMask(pkg) - if usemask.intersection(atom.use.enabled): + usemask = self.settings._getUseMask(pkg, + stable=self.settings._parent_stable) + if any(x in usemask for x in atom.use.enabled): return False - useforce = self.settings._getUseForce(pkg).difference(usemask) - if useforce.intersection(atom.use.disabled): + useforce = self.settings._getUseForce(pkg, + stable=self.settings._parent_stable) + if any(x in useforce and x not in usemask + for x in atom.use.disabled): return False + # Check unsatisfied use-default deps + if atom.use.enabled: + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.enabled for x in missing_disabled): + return False + if atom.use.disabled: + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.disabled for x in missing_enabled): + return False + return True def invalidentry(self, mypath): @@ -249,23 +302,30 @@ class dbapi(object): maxval = len(cpv_all) aux_get = self.aux_get aux_update = self.aux_update - meta_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE", 'repository'] + update_keys = Package._dep_keys + ("PROVIDE",) + meta_keys = update_keys + self._pkg_str_aux_keys repo_dict = None if isinstance(updates, dict): repo_dict = updates - from portage.update import update_dbentries if onUpdate: onUpdate(maxval, 0) if onProgress: onProgress(maxval, 0) for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - repo = metadata.pop('repository') + try: + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + except KeyError: + continue + try: + pkg = _pkg_str(cpv, metadata=metadata, settings=self.settings) + except InvalidData: + continue + metadata = dict((k, metadata[k]) for k in update_keys) if repo_dict is None: updates_list = updates else: try: - updates_list = repo_dict[repo] + updates_list = repo_dict[pkg.repo] except KeyError: try: updates_list = repo_dict['DEFAULT'] @@ -275,7 +335,8 @@ class dbapi(object): if not updates_list: continue - metadata_updates = update_dbentries(updates_list, metadata) + metadata_updates = \ + portage.update_dbentries(updates_list, metadata, parent=pkg) if metadata_updates: aux_update(cpv, metadata_updates) if onUpdate: @@ -286,27 +347,39 @@ class dbapi(object): def move_slot_ent(self, mylist, repo_match=None): """This function takes a sequence: Args: - mylist: a sequence of (package, originalslot, newslot) + mylist: a sequence of (atom, originalslot, newslot) repo_match: callable that takes single repo_name argument and returns True if the update should be applied Returns: The number of slotmoves this function did """ - pkg = mylist[1] + atom = mylist[1] origslot = mylist[2] newslot = mylist[3] - origmatches = self.match(pkg) + + try: + atom.with_slot + except AttributeError: + atom = Atom(atom).with_slot(origslot) + else: + atom = atom.with_slot(origslot) + + origmatches = self.match(atom) moves = 0 if not origmatches: return moves for mycpv in origmatches: - slot = self.aux_get(mycpv, ["SLOT"])[0] - if slot != origslot: + try: + mycpv = self._pkg_str(mycpv, atom.repo) + except (KeyError, InvalidData): continue - if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + if repo_match is not None and not repo_match(mycpv.repo): continue moves += 1 + if "/" not in newslot and \ + mycpv.sub_slot and \ + mycpv.sub_slot not in (mycpv.slot, newslot): + newslot = "%s/%s" % (newslot, mycpv.sub_slot) mydata = {"SLOT": newslot+"\n"} self.aux_update(mycpv, mydata) return moves diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.pyo b/portage_with_autodep/pym/portage/dbapi/__init__.pyo Binary files differindex e7b494d..d4d47b2 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.pyo +++ b/portage_with_autodep/pym/portage/dbapi/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py index d379b4c..9aa603d 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py @@ -1,8 +1,11 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import portage from portage.dep import Atom, _get_useflag_re +from portage.eapi import _get_eapi_attrs def expand_new_virt(vardb, atom): """ @@ -44,6 +47,7 @@ def expand_new_virt(vardb, atom): yield atom continue + eapi_attrs = _get_eapi_attrs(eapi) # Validate IUSE and IUSE, for early detection of vardb corruption. useflag_re = _get_useflag_re(eapi) valid_iuse = [] @@ -54,7 +58,11 @@ def expand_new_virt(vardb, atom): valid_iuse.append(x) valid_iuse = frozenset(valid_iuse) - iuse_implicit_match = vardb.settings._iuse_implicit_match + if eapi_attrs.iuse_effective: + iuse_implicit_match = vardb.settings._iuse_effective_match + else: + iuse_implicit_match = vardb.settings._iuse_implicit_match + valid_use = [] for x in use.split(): if x in valid_iuse or iuse_implicit_match(x): diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo Binary files differindex 6c23a7e..7884393 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.py b/portage_with_autodep/pym/portage/dbapi/bintree.py index a8027ee..b1f67ae 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.py +++ b/portage_with_autodep/pym/portage/dbapi/bintree.py @@ -1,16 +1,18 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["bindbapi", "binarytree"] import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_hash_filter', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:dep_getkey,isjustname,match_from_list', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list', 'portage.output:EOutput,colorize', 'portage.locks:lockfile,unlockfile', - 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd', 'portage.update:update_dbentries', 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ @@ -41,7 +43,9 @@ import subprocess import sys import tempfile import textwrap +import traceback import warnings +from gzip import GzipFile from itertools import chain try: from urllib.parse import urlparse @@ -49,8 +53,16 @@ except ImportError: from urlparse import urlparse if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode + +class UseCachedCopyOfRemoteIndex(Exception): + # If the local copy is recent enough + # then fetching the remote index can be skipped. + pass class bindbapi(fakedbapi): _known_keys = frozenset(list(fakedbapi._known_keys) + \ @@ -63,9 +75,10 @@ class bindbapi(fakedbapi): self.cpdict={} # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", + ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", - "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES", + "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES" ]) self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} @@ -128,15 +141,15 @@ class bindbapi(fakedbapi): if myval: mydata[x] = " ".join(myval.split()) - if not mydata.setdefault('EAPI', _unicode_decode('0')): - mydata['EAPI'] = _unicode_decode('0') + if not mydata.setdefault('EAPI', '0'): + mydata['EAPI'] = '0' if cache_me: aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, _unicode_decode('')) + aux_cache[x] = mydata.get(x, '') self._aux_cache[mycpv] = aux_cache - return [mydata.get(x, _unicode_decode('')) for x in wants] + return [mydata.get(x, '') for x in wants] def aux_update(self, cpv, values): if not self.bintree.populated: @@ -248,7 +261,7 @@ def _pkgindex_cpv_map_latest_build(pkgindex): class binarytree(object): "this tree scans for a list of all packages available in PKGDIR" - def __init__(self, _unused=None, pkgdir=None, + def __init__(self, _unused=DeprecationWarning, pkgdir=None, virtual=DeprecationWarning, settings=None): if pkgdir is None: @@ -257,11 +270,11 @@ class binarytree(object): if settings is None: raise TypeError("settings parameter is required") - if _unused is not None and _unused != settings['ROOT']: - warnings.warn("The root parameter of the " + if _unused is not DeprecationWarning: + warnings.warn("The first parameter of the " "portage.dbapi.bintree.binarytree" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) if virtual is not DeprecationWarning: @@ -293,22 +306,26 @@ class binarytree(object): self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"]) self._pkgindex_aux_keys = \ ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI", - "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", - "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", + "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", "BASE_URI"] self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) self._pkgindex_use_evaluated_keys = \ - ("LICENSE", "RDEPEND", "DEPEND", - "PDEPEND", "PROPERTIES", "PROVIDE") + ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND", + "PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT") self._pkgindex_header_keys = set([ "ACCEPT_KEYWORDS", "ACCEPT_LICENSE", - "ACCEPT_PROPERTIES", "CBUILD", + "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES", - "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"]) + "GENTOO_MIRRORS", "INSTALL_MASK", "IUSE_IMPLICIT", "USE", + "USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED"]) self._pkgindex_default_pkg_data = { "BUILD_TIME" : "", + "DEFINED_PHASES" : "", "DEPEND" : "", "EAPI" : "0", + "HDEPEND" : "", "IUSE" : "", "KEYWORDS": "", "LICENSE" : "", @@ -320,7 +337,6 @@ class binarytree(object): "RESTRICT": "", "SLOT" : "0", "USE" : "", - "DEFINED_PHASES" : "", } self._pkgindex_inherited_keys = ["CHOST", "repository"] @@ -378,15 +394,24 @@ class binarytree(object): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self.dbapi._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = portage.cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.dbapi.aux_get(mycpv, - ['repository'])[0]): + and not repo_match(mycpv.repo): + continue + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] @@ -405,7 +430,7 @@ class binarytree(object): moves += 1 mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - updated_items = update_dbentries([mylist], mydata) + updated_items = update_dbentries([mylist], mydata, parent=mycpv) mydata.update(updated_items) mydata[b'PF'] = \ _unicode_encode(mynewpkg + "\n", @@ -541,6 +566,20 @@ class binarytree(object): if not os.path.isdir(path): raise + def _file_permissions(self, path): + try: + pkgdir_st = os.stat(self.pkgdir) + except OSError: + pass + else: + pkgdir_gid = pkgdir_st.st_gid + pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode + try: + portage.util.apply_permissions(path, gid=pkgdir_gid, + mode=pkgdir_grp_mode, mask=0) + except PortageException: + pass + def _move_to_all(self, cpv): """If the file exists, move it. Whether or not it exists, update state for future getname() calls.""" @@ -796,9 +835,7 @@ class binarytree(object): del pkgindex.packages[:] pkgindex.packages.extend(iter(metadata.values())) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(self._pkgindex_file) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), @@ -841,6 +878,7 @@ class binarytree(object): if e.errno != errno.ENOENT: raise local_timestamp = pkgindex.header.get("TIMESTAMP", None) + remote_timestamp = None rmt_idx = self._new_pkgindex() proc = None tmp_filename = None @@ -849,41 +887,76 @@ class binarytree(object): # protocols and requires the base url to have a trailing # slash, so join manually... url = base_url.rstrip("/") + "/Packages" - try: - f = _urlopen(url) - except IOError: - path = parsed_url.path.rstrip("/") + "/Packages" + f = None + + # Don't use urlopen for https, since it doesn't support + # certificate/hostname verification (bug #469888). + if parsed_url.scheme not in ('https',): + try: + f = _urlopen(url, if_modified_since=local_timestamp) + if hasattr(f, 'headers') and f.headers.get('timestamp', ''): + remote_timestamp = f.headers.get('timestamp') + except IOError as err: + if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp) + raise UseCachedCopyOfRemoteIndex() + + if parsed_url.scheme in ('ftp', 'http', 'https'): + # This protocol is supposedly supported by urlopen, + # so apparently there's a problem with the url + # or a bug in urlopen. + if self.settings.get("PORTAGE_DEBUG", "0") != "0": + traceback.print_exc() - if parsed_url.scheme == 'sftp': - # The sftp command complains about 'Illegal seek' if - # we try to make it write to /dev/stdout, so use a - # temp file instead. - fd, tmp_filename = tempfile.mkstemp() - os.close(fd) - if port is not None: - port_args = ['-P', "%s" % (port,)] - proc = subprocess.Popen(['sftp'] + port_args + \ - [user_passwd + host + ":" + path, tmp_filename]) - if proc.wait() != os.EX_OK: raise - f = open(tmp_filename, 'rb') - elif parsed_url.scheme == 'ssh': + + if f is None: + + path = parsed_url.path.rstrip("/") + "/Packages" + + if parsed_url.scheme == 'ssh': + # Use a pipe so that we can terminate the download + # early if we detect that the TIMESTAMP header + # matches that of the cached Packages file. + ssh_args = ['ssh'] if port is not None: - port_args = ['-p', "%s" % (port,)] - proc = subprocess.Popen(['ssh'] + port_args + \ - [user_passwd + host, '--', 'cat', path], + ssh_args.append("-p%s" % (port,)) + # NOTE: shlex evaluates embedded quotes + ssh_args.extend(portage.util.shlex_split( + self.settings.get("PORTAGE_SSH_OPTS", ""))) + ssh_args.append(user_passwd + host) + ssh_args.append('--') + ssh_args.append('cat') + ssh_args.append(path) + + proc = subprocess.Popen(ssh_args, stdout=subprocess.PIPE) f = proc.stdout else: setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() fcmd = self.settings.get(setting) if not fcmd: - raise + fcmd = self.settings.get('FETCHCOMMAND') + if not fcmd: + raise EnvironmentError("FETCHCOMMAND is unset") + fd, tmp_filename = tempfile.mkstemp() tmp_dirname, tmp_basename = os.path.split(tmp_filename) os.close(fd) - success = portage.getbinpkg.file_get(url, - tmp_dirname, fcmd=fcmd, filename=tmp_basename) + + fcmd_vars = { + "DISTDIR": tmp_dirname, + "FILE": tmp_basename, + "URI": url + } + + for k in ("PORTAGE_SSH_OPTS",): + try: + fcmd_vars[k] = self.settings[k] + except KeyError: + pass + + success = portage.getbinpkg.file_get( + fcmd=fcmd, fcmd_vars=fcmd_vars) if not success: raise EnvironmentError("%s failed" % (setting,)) f = open(tmp_filename, 'rb') @@ -892,7 +965,8 @@ class binarytree(object): _encodings['repo.content'], errors='replace') try: rmt_idx.readHeader(f_dec) - remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) + if not remote_timestamp: # in case it had not been read from HTTP header + remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) if not remote_timestamp: # no timestamp in the header, something's wrong pkgindex = None @@ -920,6 +994,12 @@ class binarytree(object): writemsg("\n\n!!! %s\n" % \ _("Timed out while closing connection to binhost"), noiselevel=-1) + except UseCachedCopyOfRemoteIndex: + writemsg_stdout("\n") + writemsg_stdout( + colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \ + "\n") + rmt_idx = pkgindex except EnvironmentError as e: writemsg(_("\n\n!!! Error fetching binhost package" \ " info from '%s'\n") % _hide_url_passwd(base_url)) @@ -988,75 +1068,7 @@ class binarytree(object): # Local package instances override remote instances. for cpv in metadata: self._remotepkgs.pop(cpv, None) - continue - try: - chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"]) - if chunk_size < 8: - chunk_size = 8 - except (ValueError, KeyError): - chunk_size = 3000 - writemsg_stdout("\n") - writemsg_stdout( - colorize("GOOD", _("Fetching bininfo from ")) + \ - _hide_url_passwd(base_url) + "\n") - remotepkgs = portage.getbinpkg.dir_get_metadata( - base_url, chunk_size=chunk_size) - - for mypkg, remote_metadata in remotepkgs.items(): - mycat = remote_metadata.get("CATEGORY") - if mycat is None: - #old-style or corrupt package - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - mycat = mycat.strip() - try: - fullpkg = _pkg_str(mycat+"/"+mypkg[:-5]) - except InvalidData: - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - - if fullpkg in metadata: - # When using this old protocol, comparison with the remote - # package isn't supported, so the local package is always - # preferred even if getbinpkgsonly is enabled. - continue - - if not self.dbapi._category_re.match(mycat): - writemsg(_("!!! Remote binary package has an " \ - "unrecognized category: '%s'\n") % fullpkg, - noiselevel=-1) - writemsg(_("!!! '%s' has a category that is not" \ - " listed in %setc/portage/categories\n") % \ - (fullpkg, self.settings["PORTAGE_CONFIGROOT"]), - noiselevel=-1) - continue - mykey = portage.cpv_getkey(fullpkg) - try: - # invalid tbz2's can hurt things. - self.dbapi.cpv_inject(fullpkg) - for k, v in remote_metadata.items(): - remote_metadata[k] = v.strip() - remote_metadata["BASE_URI"] = base_url - - # Eliminate metadata values with names that digestCheck - # uses, since they are not valid when using the old - # protocol. Typically this is needed for SIZE metadata - # which corresponds to the size of the unpacked files - # rather than the binpkg file size, triggering digest - # verification failures as reported in bug #303211. - remote_metadata.pop('SIZE', None) - for k in portage.checksum.hashfunc_map: - remote_metadata.pop(k, None) - - self._remotepkgs[fullpkg] = remote_metadata - except SystemExit as e: - raise - except: - writemsg(_("!!! Failed to inject remote binary package: %s\n") % fullpkg, - noiselevel=-1) - continue + self.populated=1 def inject(self, cpv, filename=None): @@ -1110,6 +1122,10 @@ class binarytree(object): if not samefile: self._ensure_dir(os.path.dirname(new_filename)) _movefile(filename, new_filename, mysettings=self.settings) + full_path = new_filename + + self._file_permissions(full_path) + if self._all_directory and \ self.getname(cpv).split(os.path.sep)[-2] == "All": self._create_symlink(cpv) @@ -1157,13 +1173,35 @@ class binarytree(object): pkgindex.packages.append(d) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) + finally: if pkgindex_lock: unlockfile(pkgindex_lock) + def _pkgindex_write(self, pkgindex): + contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) + pkgindex.write(contents) + contents = contents.getvalue() + atime = mtime = long(pkgindex.header["TIMESTAMP"]) + output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"), + self._pkgindex_file, None)] + + if "compress-index" in self.settings.features: + gz_fname = self._pkgindex_file + ".gz" + fileobj = atomic_ofstream(gz_fname, mode="wb") + output_files.append((GzipFile(filename='', mode="wb", + fileobj=fileobj, mtime=mtime), gz_fname, fileobj)) + + for f, fname, f_close in output_files: + f.write(contents) + f.close() + if f_close is not None: + f_close.close() + self._file_permissions(fname) + # some seconds might have elapsed since TIMESTAMP + os.utime(fname, (atime, mtime)) + def _pkgindex_entry(self, cpv): """ Performs checksums and evaluates USE flag conditionals. @@ -1223,6 +1261,16 @@ class binarytree(object): else: header.pop(k, None) + # These values may be useful for using a binhost without + # having a local copy of the profile (bug #470006). + for k in self.settings.get("USE_EXPAND_IMPLICIT", "").split(): + k = "USE_EXPAND_VALUES_" + k + v = self.settings.get(k) + if v: + header[k] = v + else: + header.pop(k, None) + def _pkgindex_version_supported(self, pkgindex): version = pkgindex.header.get("VERSION") if version: @@ -1235,11 +1283,6 @@ class binarytree(object): def _eval_use_flags(self, cpv, metadata): use = frozenset(metadata["USE"].split()) - raw_use = use - iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split()) - use = [f for f in use if f in iuse] - use.sort() - metadata["USE"] = " ".join(use) for k in self._pkgindex_use_evaluated_keys: if k.endswith('DEPEND'): token_class = Atom @@ -1248,7 +1291,7 @@ class binarytree(object): try: deps = metadata[k] - deps = use_reduce(deps, uselist=raw_use, token_class=token_class) + deps = use_reduce(deps, uselist=use, token_class=token_class) deps = paren_enclose(deps) except portage.exception.InvalidDependString as e: writemsg("%s: %s\n" % (k, str(e)), @@ -1313,6 +1356,8 @@ class binarytree(object): """Returns the URI to the Packages file for a given package.""" return self._pkgindex_uri.get(pkgname) + + def gettbz2(self, pkgname): """Fetches the package from a remote site, if necessary. Attempts to resume if the file appears to be partially downloaded.""" @@ -1320,7 +1365,7 @@ class binarytree(object): tbz2name = os.path.basename(tbz2_path) resume = False if os.path.exists(tbz2_path): - if (tbz2name not in self.invalids): + if tbz2name[:-5] not in self.invalids: return else: resume = True @@ -1370,19 +1415,14 @@ class binarytree(object): f.close() return pkgindex - def digestCheck(self, pkg): - """ - Verify digests for the given package and raise DigestException - if verification fails. - @rtype: bool - @return: True if digests could be located, False otherwise. - """ - cpv = pkg - if not isinstance(cpv, basestring): + def _get_digests(self, pkg): + + try: cpv = pkg.cpv - pkg = None + except AttributeError: + cpv = pkg - pkg_path = self.getname(cpv) + digests = {} metadata = None if self._remotepkgs is None or cpv not in self._remotepkgs: for d in self._load_pkgindex().packages: @@ -1392,9 +1432,8 @@ class binarytree(object): else: metadata = self._remotepkgs[cpv] if metadata is None: - return False + return digests - digests = {} for k in hashfunc_map: v = metadata.get(k) if not v: @@ -1408,9 +1447,31 @@ class binarytree(object): writemsg(_("!!! Malformed SIZE attribute in remote " \ "metadata for '%s'\n") % cpv) + return digests + + def digestCheck(self, pkg): + """ + Verify digests for the given package and raise DigestException + if verification fails. + @rtype: bool + @return: True if digests could be located, False otherwise. + """ + + digests = self._get_digests(pkg) + if not digests: return False + try: + cpv = pkg.cpv + except AttributeError: + cpv = pkg + + pkg_path = self.getname(cpv) + hash_filter = _hash_filter( + self.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if not hash_filter.transparent: + digests = _apply_hash_filter(digests, hash_filter) eout = EOutput() eout.quiet = self.settings.get("PORTAGE_QUIET") == "1" ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0) @@ -1426,9 +1487,7 @@ class binarytree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0] - except SystemExit as e: - raise - except Exception as e: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.pyo b/portage_with_autodep/pym/portage/dbapi/bintree.pyo Binary files differindex f99f377..90d0c6a 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/bintree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py index 947194c..70ee782 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["cpv_expand"] import portage diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo Binary files differindex cf1a428..7c38720 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.py b/portage_with_autodep/pym/portage/dbapi/dep_expand.py index ac8ccf4..3de5d8f 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dep_expand"] import re @@ -23,7 +25,7 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None): if mydep[0] == "*": mydep = mydep[1:] orig_dep = mydep - has_cat = '/' in orig_dep + has_cat = '/' in orig_dep.split(':')[0] if not has_cat: alphanum = re.search(r'\w', orig_dep) if alphanum: diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo Binary files differindex b323f5b..bcaf8e3 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.py b/portage_with_autodep/pym/portage/dbapi/porttree.py index c5ee770..fc3fc03 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.py +++ b/portage_with_autodep/pym/portage/dbapi/porttree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi" ] @@ -10,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce', + 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', @@ -22,7 +24,8 @@ from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping from portage.dbapi import dbapi from portage.exception import PortageException, \ - FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName + FileNotFound, InvalidAtom, InvalidData, \ + InvalidDependString, InvalidPackageName from portage.localization import _ from portage import eclass_cache, \ @@ -32,21 +35,74 @@ from portage import os from portage import _encodings from portage import _unicode_encode from portage import OrderedDict +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildMetadataPhase import EbuildMetadataPhase -from _emerge.PollScheduler import PollScheduler import os as _os import sys import traceback import warnings +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + if sys.hexversion >= 0x3000000: basestring = str long = int +def close_portdbapi_caches(): + # The python interpreter does _not_ guarantee that destructors are + # called for objects that remain when the interpreter exits, so we + # use an atexit hook to call destructors for any global portdbapi + # instances that may have been constructed. + try: + portage._legacy_globals_constructed + except AttributeError: + pass + else: + if "db" in portage._legacy_globals_constructed: + try: + db = portage.db + except AttributeError: + pass + else: + if isinstance(db, dict): + for x in db.values(): + try: + if "porttree" in x.lazy_items: + continue + except (AttributeError, TypeError): + continue + try: + x = x.pop("porttree").dbapi + except (AttributeError, KeyError): + continue + if not isinstance(x, portdbapi): + continue + x.close_caches() + +portage.process.atexit_register(close_portdbapi_caches) + +# It used to be necessary for API consumers to remove portdbapi instances +# from portdbapi_instances, in order to avoid having accumulated instances +# consume memory. Now, portdbapi_instances is just an empty dummy list, so +# for backward compatibility, ignore ValueError for removal on non-existent +# items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy portdbapi_instances. + try: + list.remove(self, item) + except ValueError: + pass + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" - portdbapi_instances = [] + portdbapi_instances = _dummy_list() _use_mutable = True @property @@ -64,14 +120,13 @@ class portdbapi(dbapi): return None return main_repo.eclass_db - def __init__(self, _unused_param=None, mysettings=None): + def __init__(self, _unused_param=DeprecationWarning, mysettings=None): """ @param _unused_param: deprecated, use mysettings['PORTDIR'] instead @type _unused_param: None @param mysettings: an immutable config instance @type mysettings: portage.config """ - portdbapi.portdbapi_instances.append(self) from portage import config if mysettings: @@ -80,7 +135,7 @@ class portdbapi(dbapi): from portage import settings self.settings = config(clone=settings) - if _unused_param is not None: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " + \ "portage.dbapi.porttree.portdbapi" + \ " constructor is unused since portage-2.1.8. " + \ @@ -95,7 +150,6 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.settings) - self._scheduler = PollScheduler().sched_iface self.depcachedir = os.path.realpath(self.settings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -152,10 +206,10 @@ class portdbapi(dbapi): # portage group. depcachedir_unshared = True else: - cache_kwargs.update({ + cache_kwargs.update(portage._native_kwargs({ 'gid' : portage_gid, 'perms' : 0o664 - }) + })) # If secpass < 1, we don't want to write to the cache # since then we won't be able to apply group permissions @@ -186,13 +240,25 @@ class portdbapi(dbapi): self._pregen_auxdb[x] = cache # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", + ["DEPEND", "EAPI", "HDEPEND", + "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"]) self._aux_cache = {} self._broken_ebuilds = set() + @property + def _event_loop(self): + if portage._internal_caller: + # For internal portage usage, the global_event_loop is safe. + return global_event_loop() + else: + # For external API consumers, use a local EventLoop, since + # we don't want to assume that it's safe to override the + # global SIGCHLD handler. + return EventLoop(main=False) + def _create_pregen_cache(self, tree): conf = self.repositories.get_repo_for_location(tree) cache = conf.get_pregenerated_cache( @@ -202,6 +268,13 @@ class portdbapi(dbapi): cache.ec = self.repositories.get_repo_for_location(tree).eclass_db except AttributeError: pass + + if not cache.complete_eclass_entries: + warnings.warn( + ("Repository '%s' used deprecated 'pms' cache format. " + "Please migrate to 'md5-dict' format.") % (conf.name,), + DeprecationWarning) + return cache def _init_cache_dirs(self): @@ -446,7 +519,7 @@ class portdbapi(dbapi): proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, - repo_path=mylocation, scheduler=self._scheduler, + repo_path=mylocation, scheduler=self._event_loop, settings=self.doebuild_settings) proc.start() @@ -626,13 +699,14 @@ class portdbapi(dbapi): else: return 0 - def cp_all(self, categories=None, trees=None): + def cp_all(self, categories=None, trees=None, reverse=False): """ This returns a list of all keys in our tree or trees @param categories: optional list of categories to search or defaults to self.settings.categories @param trees: optional list of trees to search the categories in or defaults to self.porttrees + @param reverse: reverse sort order (default is False) @rtype list of [cat/pkg,...] """ d = {} @@ -651,7 +725,7 @@ class portdbapi(dbapi): continue d[atom.cp] = None l = list(d) - l.sort() + l.sort(reverse=reverse) return l def cp_list(self, mycp, use_cache=1, mytree=None): @@ -825,18 +899,24 @@ class portdbapi(dbapi): # ebuild not in this repo, or masked by corruption continue - if visibility_filter and not self._visible(cpv, metadata): + try: + pkg_str = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + + if visibility_filter and not self._visible(pkg_str, metadata): continue if mydep.slot is not None and \ - mydep.slot != metadata["SLOT"]: + not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ - not self._match_use(mydep, cpv, metadata): + not self._match_use(mydep, pkg_str, metadata): continue - myval.append(cpv) + myval.append(pkg_str) # only yield a given cpv once break @@ -959,19 +1039,16 @@ class portdbapi(dbapi): return False if settings._getMissingProperties(cpv, metadata): return False + if settings._getMissingRestrict(cpv, metadata): + return False except InvalidDependString: return False return True -def close_portdbapi_caches(): - for i in portdbapi.portdbapi_instances: - i.close_caches() - -portage.process.atexit_register(portage.portageexit) - class portagetree(object): - def __init__(self, root=None, virtual=DeprecationWarning, settings=None): + def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning, + settings=None): """ Constructor for a PortageTree @@ -987,7 +1064,7 @@ class portagetree(object): settings = portage.settings self.settings = settings - if root is not None and root != settings['ROOT']: + if root is not DeprecationWarning: warnings.warn("The root parameter of the " + \ "portage.dbapi.porttree.portagetree" + \ " constructor is now unused. Use " + \ @@ -1055,10 +1132,8 @@ class portagetree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] - except SystemExit: - raise - except Exception: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot @@ -1130,9 +1205,18 @@ def _parse_uri_map(cpv, metadata, use=None): uri_set = uri_map.get(distfile) if uri_set is None: - uri_set = set() + # Use OrderedDict to preserve order from SRC_URI + # while ensuring uniqueness. + uri_set = OrderedDict() uri_map[distfile] = uri_set - uri_set.add(uri) - uri = None + + # SRC_URI may contain a file name with no scheme, and in + # this case it does not belong in uri_set. + if urlparse(uri).scheme: + uri_set[uri] = True + + # Convert OrderedDicts to tuples. + for k, v in uri_map.items(): + uri_map[k] = tuple(v) return uri_map diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.pyo b/portage_with_autodep/pym/portage/dbapi/porttree.pyo Binary files differindex fb57919..43ce4a8 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/porttree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.py b/portage_with_autodep/pym/portage/dbapi/vartree.py index 517c873..ed62323 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.py +++ b/portage_with_autodep/pym/portage/dbapi/vartree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "vardbapi", "vartree", "dblink"] + \ ["write_contents", "tar_contents"] @@ -11,8 +13,10 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid,portage_uid,secpass', 'portage.dbapi.dep_expand:dep_expand', 'portage.dbapi._MergeProcess:MergeProcess', - 'portage.dep:dep_getkey,isjustname,match_from_list,' + \ - 'use_reduce,_slot_re', + 'portage.dbapi._SyncfsProcess:SyncfsProcess', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ + 'use_reduce,_slot_separator,_repo_separator', + 'portage.eapi:_get_eapi_attrs', 'portage.elog:collect_ebuild_messages,collect_messages,' + \ 'elog_process,_merge_logentries', 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', @@ -20,7 +24,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.doebuild:doebuild_environment,' + \ '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', - 'portage.update:fixdbentries', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', @@ -30,15 +34,17 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.movefile:movefile', 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ - '_pkgsplit@pkgsplit,_pkg_str', + '_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo', 'subprocess', 'tarfile', ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_PRESERVE_LIBS from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ @@ -59,8 +65,8 @@ from portage import _unicode_encode from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog -from _emerge.PollScheduler import PollScheduler from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.SpawnProcess import SpawnProcess import errno import fnmatch @@ -70,6 +76,7 @@ import io from itertools import chain import logging import os as _os +import platform import pwd import re import stat @@ -108,7 +115,8 @@ class vardbapi(dbapi): _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') - def __init__(self, _unused_param=None, categories=None, settings=None, vartree=None): + def __init__(self, _unused_param=DeprecationWarning, + categories=None, settings=None, vartree=None): """ The categories parameter is unused since the dbapi class now has a categories property that is generated from the @@ -138,11 +146,11 @@ class vardbapi(dbapi): settings = portage.settings self.settings = settings - if _unused_param is not None and _unused_param != settings['ROOT']: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " "portage.dbapi.vartree.vardbapi" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) self._eroot = settings['EROOT'] @@ -159,7 +167,7 @@ class vardbapi(dbapi): self.vartree = vartree self._aux_cache_keys = set( ["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION", - "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", + "EAPI", "HDEPEND", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES", ]) @@ -169,15 +177,9 @@ class vardbapi(dbapi): self._counter_path = os.path.join(self._eroot, CACHE_PATH, "counter") - self._plib_registry = None - if _ENABLE_PRESERVE_LIBS: - self._plib_registry = PreservedLibsRegistry(settings["ROOT"], - os.path.join(self._eroot, PRIVATE_PATH, - "preserved_libs_registry")) - - self._linkmap = None - if _ENABLE_DYN_LINK_MAP: - self._linkmap = LinkageMap(self) + self._plib_registry = PreservedLibsRegistry(settings["ROOT"], + os.path.join(self._eroot, PRIVATE_PATH, "preserved_libs_registry")) + self._linkmap = LinkageMap(self) self._owners = self._owners_db(self) self._cached_counter = None @@ -318,14 +320,24 @@ class vardbapi(dbapi): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + and not repo_match(mycpv.repo): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): + continue + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): @@ -355,7 +367,7 @@ class vardbapi(dbapi): del e write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - fixdbentries([mylist], newpath) + return moves def cp_list(self, mycp, use_cache=1): @@ -363,7 +375,10 @@ class vardbapi(dbapi): if mysplit[0] == '*': mysplit[0] = mysplit[0][1:] try: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime + if sys.hexversion >= 0x3030000: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns + else: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime except OSError: mystat = 0 if use_cache and mycp in self.cpcache: @@ -498,7 +513,10 @@ class vardbapi(dbapi): return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime + if sys.hexversion >= 0x3030000: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns + else: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime except (IOError, OSError): curmtime=0 @@ -553,31 +571,32 @@ class vardbapi(dbapi): def _aux_cache_init(self): aux_cache = None open_kwargs = {} - if sys.hexversion >= 0x3000000: + if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000: # Buffered io triggers extreme performance issues in # Unpickler.load() (problem observed with python-3.0.1). # Unfortunately, performance is still poor relative to - # python-2.x, but buffering makes it much worse. + # python-2.x, but buffering makes it much worse (problem + # appears to be solved in Python >=3.2 at least). open_kwargs["buffering"] = 0 try: - f = open(_unicode_encode(self._aux_cache_filename, + with open(_unicode_encode(self._aux_cache_filename, encoding=_encodings['fs'], errors='strict'), - mode='rb', **open_kwargs) - mypickle = pickle.Unpickler(f) - try: - mypickle.find_global = None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). - pass - aux_cache = mypickle.load() - f.close() - del f - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + mode='rb', **open_kwargs) as f: + mypickle = pickle.Unpickler(f) + try: + mypickle.find_global = None + except AttributeError: + # TODO: If py3k, override Unpickler.find_class(). + pass + aux_cache = mypickle.load() + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: if isinstance(e, EnvironmentError) and \ getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass else: - writemsg(_unicode_decode(_("!!! Error loading '%s': %s\n")) % \ + writemsg(_("!!! Error loading '%s': %s\n") % \ (self._aux_cache_filename, e), noiselevel=-1) del e @@ -644,7 +663,8 @@ class vardbapi(dbapi): if e.errno != errno.ENOENT: raise raise KeyError(mycpv) - mydir_mtime = mydir_stat[stat.ST_MTIME] + # Use float mtime when available. + mydir_mtime = mydir_stat.st_mtime pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) mydata = {"_mtime_" : mydir_mtime} @@ -657,13 +677,18 @@ class vardbapi(dbapi): pkg_data = None else: cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (long, int)) or \ + if not isinstance(cache_mtime, (float, long, int)) or \ not isinstance(metadata, dict): pkg_data = None if pkg_data: cache_mtime, metadata = pkg_data - cache_valid = cache_mtime == mydir_mtime + if isinstance(cache_mtime, float): + cache_valid = cache_mtime == mydir_stat.st_mtime + else: + # Cache may contain integer mtime. + cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME] + if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.items(): @@ -687,10 +712,11 @@ class vardbapi(dbapi): (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) - if _slot_re.match(mydata['SLOT']) is None: + eapi_attrs = _get_eapi_attrs(mydata['EAPI']) + if _get_slot_re(eapi_attrs).match(mydata['SLOT']) is None: # Empty or invalid slot triggers InvalidAtom exceptions when # generating slot atoms for packages, so translate it to '0' here. - mydata['SLOT'] = _unicode_decode('0') + mydata['SLOT'] = '0' return [mydata[x] for x in wants] @@ -715,21 +741,18 @@ class vardbapi(dbapi): results[x] = st[stat.ST_MTIME] continue try: - myf = io.open( + with io.open( _unicode_encode(os.path.join(mydir, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: - myd = myf.read() - finally: - myf.close() + errors='replace') as f: + myd = f.read() except IOError: if x not in self._aux_cache_keys and \ self._aux_cache_keys_re.match(x) is None: env_keys.append(x) continue - myd = _unicode_decode('') + myd = '' # Preserve \n for metadata that is known to # contain multiple lines. @@ -743,13 +766,13 @@ class vardbapi(dbapi): for k in env_keys: v = env_results.get(k) if v is None: - v = _unicode_decode('') + v = '' if self._aux_multi_line_re.match(k) is None: v = " ".join(v.split()) results[k] = v if results.get("EAPI") == "": - results[_unicode_decode("EAPI")] = _unicode_decode('0') + results["EAPI"] = '0' return results @@ -869,11 +892,17 @@ class vardbapi(dbapi): del myroot counter = -1 try: - cfile = io.open( + with io.open( _unicode_encode(self._counter_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + try: + counter = long(f.readline().strip()) + except (OverflowError, ValueError) as e: + writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % + self._counter_path, noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) except EnvironmentError as e: # Silently allow ENOENT since files under # /var/cache/ are allowed to disappear. @@ -882,17 +911,6 @@ class vardbapi(dbapi): self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) del e - else: - try: - try: - counter = long(cfile.readline().strip()) - finally: - cfile.close() - except (OverflowError, ValueError) as e: - writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % \ - self._counter_path, noiselevel=-1) - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e if self._cached_counter == counter: max_counter = counter @@ -984,16 +1002,31 @@ class vardbapi(dbapi): relative_filename = filename[root_len:] contents_key = pkg._match_contents(relative_filename) if contents_key: - del new_contents[contents_key] + # It's possible for two different paths to refer to the same + # contents_key, due to directory symlinks. Therefore, pass a + # default value to pop, in order to avoid a KeyError which + # could otherwise be triggered (see bug #454400). + new_contents.pop(contents_key, None) removed += 1 if removed: - self._bump_mtime(pkg.mycpv) - f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) - write_contents(new_contents, root, f) - f.close() - self._bump_mtime(pkg.mycpv) - pkg._clear_contents_cache() + self.writeContentsToContentsFile(pkg, new_contents) + + def writeContentsToContentsFile(self, pkg, new_contents): + """ + @param pkg: package to write contents file for + @type pkg: dblink + @param new_contents: contents to write to CONTENTS file + @type new_contents: contents dictionary of the form + {u'/path/to/file' : (contents_attribute 1, ...), ...} + """ + root = self.settings['ROOT'] + self._bump_mtime(pkg.mycpv) + f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) + write_contents(new_contents, root, f) + f.close() + self._bump_mtime(pkg.mycpv) + pkg._clear_contents_cache() class _owners_cache(object): """ @@ -1238,18 +1271,35 @@ class vardbapi(dbapi): name = os.path.basename(path.rstrip(os.path.sep)) path_info_list.append((path, name, is_basename)) + # Do work via the global event loop, so that it can be used + # for indication of progress during the search (bug #461412). + event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) root = self._vardb._eroot - for cpv in self._vardb.cpv_all(): - dblnk = self._vardb._dblink(cpv) + def search_pkg(cpv): + dblnk = self._vardb._dblink(cpv) for path, name, is_basename in path_info_list: if is_basename: for p in dblnk.getcontents(): if os.path.basename(p) == name: - yield dblnk, p[len(root):] + search_pkg.results.append((dblnk, p[len(root):])) else: if dblnk.isowner(path): - yield dblnk, path + search_pkg.results.append((dblnk, path)) + search_pkg.complete = True + return False + + search_pkg.results = [] + + for cpv in self._vardb.cpv_all(): + del search_pkg.results[:] + search_pkg.complete = False + event_loop.idle_add(search_pkg, cpv) + while not search_pkg.complete: + event_loop.iteration() + for result in search_pkg.results: + yield result class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" @@ -1370,7 +1420,7 @@ class vartree(object): def getslot(self, mycatpkg): "Get a slot for a catpkg; assume it exists." try: - return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + return self.dbapi._pkg_str(mycatpkg, None).slot except KeyError: return "" @@ -1463,11 +1513,16 @@ class dblink(object): self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False + self._device_path_map = {} self._hardlink_merge_map = {} self._hash_key = (self._eroot, self.mycpv) self._protect_obj = None self._pipe = pipe + # When necessary, this attribute is modified for + # compliance with RESTRICT=preserve-libs. + self._preserve_libs = "preserve-libs" in mysettings.features + def __hash__(self): return hash(self._hash_key) @@ -1510,7 +1565,11 @@ class dblink(object): """ Remove this entry from the database """ - if not os.path.exists(self.dbdir): + try: + os.lstat(self.dbdir) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE): + raise return # Check validity of self.dbdir before attempting to remove it. @@ -1527,6 +1586,14 @@ class dblink(object): pass self.vartree.dbapi._remove(self) + # Use self.dbroot since we need an existing path for syncfs. + try: + self._merged_path(self.dbroot, os.lstat(self.dbroot)) + except OSError: + pass + + self._post_merge_sync() + def clearcontents(self): """ For a given db entry (self), erase the CONTENTS values. @@ -1552,18 +1619,18 @@ class dblink(object): return self.contentscache pkgfiles = {} try: - myc = io.open(_unicode_encode(contents_file, + with io.open(_unicode_encode(contents_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + mylines = f.readlines() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e self.contentscache = pkgfiles return pkgfiles - mylines = myc.readlines() - myc.close() + null_byte = "\0" normalize_needed = self._normalize_needed contents_re = self._contents_re @@ -1578,7 +1645,7 @@ class dblink(object): if myroot == os.path.sep: myroot = None # used to generate parent dir entries - dir_entry = (_unicode_decode("dir"),) + dir_entry = ("dir",) eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1 pos = 0 errors = [] @@ -1678,8 +1745,11 @@ class dblink(object): unmerge_preserve = \ self._find_libs_to_preserve(unmerge=True) counter = self.vartree.dbapi.cpv_counter(self.mycpv) - plib_registry.unregister(self.mycpv, - self.settings["SLOT"], counter) + try: + slot = self.mycpv.slot + except AttributeError: + slot = _pkg_str(self.mycpv, slot=self.settings["SLOT"]).slot + plib_registry.unregister(self.mycpv, slot, counter) if unmerge_preserve: for path in sorted(unmerge_preserve): contents_key = self._match_contents(path) @@ -1689,7 +1759,7 @@ class dblink(object): self._display_merge(_(">>> needed %s %s\n") % \ (obj_type, contents_key), noiselevel=-1) plib_registry.register(self.mycpv, - self.settings["SLOT"], counter, unmerge_preserve) + slot, counter, unmerge_preserve) # Remove the preserved files from our contents # so that they won't be unmerged. self.vartree.dbapi.removeFromContents(self, @@ -1759,7 +1829,8 @@ class dblink(object): if self._scheduler is None: # We create a scheduler instance and use it to # log unmerge output separately from merge output. - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) if self.settings.get("PORTAGE_BACKGROUND") == "subprocess": if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1": self.settings["PORTAGE_BACKGROUND"] = "1" @@ -1775,11 +1846,16 @@ class dblink(object): showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None + + # When others_in_slot is not None, the backup has already been + # handled by the caller. + caller_handles_backup = others_in_slot is not None + # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. if others_in_slot is None: - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) others_in_slot = [] @@ -1823,8 +1899,9 @@ class dblink(object): except UnsupportedAPIException as e: eapi_unsupported = e - self._prune_plib_registry(unmerge=True, needed=needed, - preserve_paths=preserve_paths) + if self._preserve_libs and "preserve-libs" in \ + self.settings["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False builddir_lock = None scheduler = self._scheduler @@ -1832,7 +1909,7 @@ class dblink(object): try: # Only create builddir_lock if the caller # has not already acquired the lock. - if "PORTAGE_BUILDIR_LOCKED" not in self.settings: + if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: builddir_lock = EbuildBuildDir( scheduler=scheduler, settings=self.settings) @@ -1840,6 +1917,19 @@ class dblink(object): prepare_build_dirs(settings=self.settings, cleanup=True) log_path = self.settings.get("PORTAGE_LOG_FILE") + # Do this before the following _prune_plib_registry call, since + # that removes preserved libraries from our CONTENTS, and we + # may want to backup those libraries first. + if not caller_handles_backup: + retval = self._pre_unmerge_backup(background) + if retval != os.EX_OK: + showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval, + level=logging.ERROR, noiselevel=-1) + return retval + + self._prune_plib_registry(unmerge=True, needed=needed, + preserve_paths=preserve_paths) + # Log the error after PORTAGE_LOG_FILE is initialized # by prepare_build_dirs above. if eapi_unsupported: @@ -1848,7 +1938,7 @@ class dblink(object): showMessage(_("!!! FAILED prerm: %s\n") % \ os.path.join(self.dbdir, "EAPI"), level=logging.ERROR, noiselevel=-1) - showMessage(_unicode_decode("%s\n") % (eapi_unsupported,), + showMessage("%s\n" % (eapi_unsupported,), level=logging.ERROR, noiselevel=-1) elif os.path.isfile(myebuildpath): phase = EbuildPhase(background=background, @@ -2037,7 +2127,7 @@ class dblink(object): if others_in_slot is None: others_in_slot = [] - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) for cur_cpv in slot_matches: @@ -2062,7 +2152,9 @@ class dblink(object): #process symlinks second-to-last, directories last. mydirs = set() - modprotect = os.path.join(self._eroot, "lib/modules/") + + uninstall_ignore = portage.util.shlex_split( + self.settings.get("UNINSTALL_IGNORE", "")) def unlink(file_name, lstatobj): if bsd_chflags: @@ -2092,6 +2184,14 @@ class dblink(object): self._eerror("postrm", ["Could not chmod or unlink '%s': %s" % \ (file_name, ose)]) + else: + + # Even though the file no longer exists, we log it + # here so that _unmerge_dirs can see that we've + # removed a file from this device, and will record + # the parent directory for a syncfs call. + self._merged_path(file_name, lstatobj, exists=False) + finally: if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking @@ -2169,6 +2269,24 @@ class dblink(object): if lstatobj is None: show_unmerge("---", unmerge_desc["!found"], file_type, obj) continue + + f_match = obj[len(eroot)-1:] + ignore = False + for pattern in uninstall_ignore: + if fnmatch.fnmatch(f_match, pattern): + ignore = True + break + + if not ignore: + if islink and f_match in \ + ("/lib", "/usr/lib", "/usr/local/lib"): + # Ignore libdir symlinks for bug #423127. + ignore = True + + if ignore: + show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) + continue + # don't use EROOT, CONTENTS entries already contain EPREFIX if obj.startswith(real_root): relative_path = obj[real_root_len:] @@ -2178,8 +2296,9 @@ class dblink(object): is_owned = True break - if file_type == "sym" and is_owned and \ - (islink and statobj and stat.S_ISDIR(statobj.st_mode)): + if is_owned and islink and \ + file_type in ("sym", "dir") and \ + statobj and stat.S_ISDIR(statobj.st_mode): # A new instance of this package claims the file, so # don't unmerge it. If the file is symlink to a # directory and the unmerging package installed it as @@ -2211,18 +2330,6 @@ class dblink(object): continue elif relative_path in cfgfiledict: stale_confmem.append(relative_path) - # next line includes a tweak to protect modules from being unmerged, - # but we don't protect modules from being overwritten if they are - # upgraded. We effectively only want one half of the config protection - # functionality for /lib/modules. For portage-ng both capabilities - # should be able to be independently specified. - # TODO: For rebuilds, re-parent previous modules to the new - # installed instance (so they are not orphans). For normal - # uninstall (not rebuild/reinstall), remove the modules along - # with all other files (leave no orphans). - if obj.startswith(modprotect): - show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) - continue # Don't unlink symlinks to directories here since that can # remove /lib and /usr/lib symlinks. @@ -2244,12 +2351,12 @@ class dblink(object): show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) continue - if pkgfiles[objkey][0] == "dir": + if file_type == "dir" and not islink: if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): show_unmerge("---", unmerge_desc["!dir"], file_type, obj) continue mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino))) - elif pkgfiles[objkey][0] == "sym": + elif file_type == "sym" or (file_type == "dir" and islink): if not islink: show_unmerge("---", unmerge_desc["!sym"], file_type, obj) continue @@ -2359,7 +2466,11 @@ class dblink(object): if protected_symlinks: msg = "One or more symlinks to directories have been " + \ "preserved in order to ensure that files installed " + \ - "via these symlinks remain accessible:" + "via these symlinks remain accessible. " + \ + "This indicates that the mentioned symlink(s) may " + \ + "be obsolete remnants of an old install, and it " + \ + "may be appropriate to replace a given symlink " + \ + "with the directory that it points to." lines = textwrap.wrap(msg, 72) lines.append("") flat_list = set() @@ -2369,7 +2480,7 @@ class dblink(object): lines.append("\t%s" % (os.path.join(real_root, f.lstrip(os.sep)))) lines.append("") - self._elog("eerror", "postrm", lines) + self._elog("elog", "postrm", lines) # Remove stale entries from config memory. if stale_confmem: @@ -2501,15 +2612,19 @@ class dblink(object): raise del e show_unmerge("!!!", "", "obj", child) + try: + parent_name = os.path.dirname(obj) + parent_stat = os.stat(parent_name) + if bsd_chflags: lstatobj = os.lstat(obj) if lstatobj.st_flags != 0: bsd_chflags.lchflags(obj, 0) - parent_name = os.path.dirname(obj) + # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. - pflags = os.stat(parent_name).st_flags + pflags = parent_stat.st_flags if pflags != 0: bsd_chflags.chflags(parent_name, 0) try: @@ -2518,13 +2633,34 @@ class dblink(object): if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) + + # Record the parent directory for use in syncfs calls. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real parent directory resides. + self._merged_path(os.path.realpath(parent_name), parent_stat) + show_unmerge("<<<", "", "dir", obj) except EnvironmentError as e: if e.errno not in ignored_rmdir_errnos: raise if e.errno != errno.ENOENT: show_unmerge("---", unmerge_desc["!empty"], "dir", obj) - del e + + # Since we didn't remove this directory, record the directory + # itself for use in syncfs calls, if we have removed another + # file from the same device. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real directory resides. + try: + dir_stat = os.stat(obj) + except OSError: + pass + else: + if dir_stat.st_dev in self._device_path_map: + self._merged_path(os.path.realpath(obj), dir_stat) + else: # When a directory is successfully removed, there's # no need to protect symlinks that point to it. @@ -2751,7 +2887,7 @@ class dblink(object): self.vartree.dbapi._linkmap is None or \ self.vartree.dbapi._plib_registry is None or \ (not unmerge and self._installed_instance is None) or \ - "preserve-libs" not in self.settings.features: + not self._preserve_libs: return set() os = _os_merge @@ -3335,7 +3471,10 @@ class dblink(object): else: logdir = os.path.join(self.settings["T"], "logging") ebuild_logentries = collect_ebuild_messages(logdir) - py_logentries = collect_messages(key=cpv).get(cpv, {}) + # phasefilter is irrelevant for the above collect_ebuild_messages + # call, since this package instance has a private logdir. However, + # it may be relevant for the following collect_messages call. + py_logentries = collect_messages(key=cpv, phasefilter=phasefilter).get(cpv, {}) logentries = _merge_logentries(py_logentries, ebuild_logentries) funcnames = { "INFO": "einfo", @@ -3356,7 +3495,9 @@ class dblink(object): str_buffer.append(' '.join(fields)) str_buffer.append('\n') if str_buffer: - os.write(self._pipe, _unicode_encode(''.join(str_buffer))) + str_buffer = _unicode_encode(''.join(str_buffer)) + while str_buffer: + str_buffer = str_buffer[os.write(self._pipe, str_buffer):] def _emerge_log(self, msg): emergelog(False, msg) @@ -3414,6 +3555,7 @@ class dblink(object): level=logging.ERROR, noiselevel=-1) return 1 + is_binpkg = self.settings.get("EMERGE_FROM") == "binary" slot = '' for var_name in ('CHOST', 'SLOT'): if var_name == 'CHOST' and self.cat == 'virtual': @@ -3423,22 +3565,18 @@ class dblink(object): pass continue - f = None try: - f = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(inforoot, var_name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - val = f.readline().strip() + errors='replace') as f: + val = f.readline().strip() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e val = '' - finally: - if f is not None: - f.close() if var_name == 'SLOT': slot = val @@ -3451,7 +3589,9 @@ class dblink(object): return 1 write_atomic(os.path.join(inforoot, var_name), slot + '\n') - if val != self.settings.get(var_name, ''): + # This check only applies when built from source, since + # inforoot values are written just after src_install. + if not is_binpkg and val != self.settings.get(var_name, ''): self._eqawarn('preinst', [_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \ {"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}]) @@ -3462,30 +3602,47 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) + # NOTE: We use SLOT obtained from the inforoot + # directory, in order to support USE=multislot. + # Use _pkg_str discard the sub-slot part if necessary. + slot = _pkg_str(self.mycpv, slot=slot).slot cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) - # filter any old-style virtual matches - slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) \ - if cpv_getkey(cpv) == cp] - - if self.mycpv not in slot_matches and \ - self.vartree.dbapi.cpv_exists(self.mycpv): - # handle multislot or unapplied slotmove - slot_matches.append(self.mycpv) - - others_in_slot = [] - from portage import config - for cur_cpv in slot_matches: - # Clone the config in case one of these has to be unmerged since - # we need it to have private ${T} etc... for things like elog. - settings_clone = config(clone=self.settings) - settings_clone.pop("PORTAGE_BUILDIR_LOCKED", None) - settings_clone.reset() - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=settings_clone, - vartree=self.vartree, treetype="vartree", - scheduler=self._scheduler, pipe=self._pipe)) + self.lockdb() + try: + # filter any old-style virtual matches + slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) + if cpv_getkey(cpv) == cp] + + if self.mycpv not in slot_matches and \ + self.vartree.dbapi.cpv_exists(self.mycpv): + # handle multislot or unapplied slotmove + slot_matches.append(self.mycpv) + + others_in_slot = [] + for cur_cpv in slot_matches: + # Clone the config in case one of these has to be unmerged, + # since we need it to have private ${T} etc... for things + # like elog. + settings_clone = portage.config(clone=self.settings) + settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None) + settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi) + if self._preserve_libs and "preserve-libs" in \ + settings_clone["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False + others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], + settings=settings_clone, + vartree=self.vartree, treetype="vartree", + scheduler=self._scheduler, pipe=self._pipe)) + finally: + self.unlockdb() + + # If any instance has RESTRICT=preserve-libs, then + # restrict it for all instances. + if not self._preserve_libs: + for dblnk in others_in_slot: + dblnk._preserve_libs = False retval = self._security_check(others_in_slot) if retval: @@ -3596,6 +3753,13 @@ class dblink(object): # to an infinite recursion loop. mylinklist.append(relative_path) + myto = _unicode_decode( + _os.readlink(_unicode_encode(fpath, + encoding=_encodings['merge'], errors='strict')), + encoding=_encodings['merge'], errors='replace') + if line_ending_re.search(myto) is not None: + paths_with_newlines.append(relative_path) + if unicode_error: break @@ -3647,7 +3811,7 @@ class dblink(object): _("Manually run `emerge --unmerge =%s` if you " "really want to remove the above files. Set " "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " - "/etc/make.conf if you do not want to " + "/etc/portage/make.conf if you do not want to " "abort in cases like this.") % other_dblink.mycpv, wrap_width)) eerror(msg) @@ -3713,7 +3877,9 @@ class dblink(object): " enough information to determine if a real problem" " exists. Please do NOT file a bug report at" " http://bugs.gentoo.org unless you report exactly which" - " two packages install the same file(s). Once again," + " two packages install the same file(s). See" + " http://wiki.gentoo.org/wiki/Knowledge_Base:Blockers" + " for tips on how to solve the problem. And once again," " please do NOT file a bug report unless you have" " completely understood the above message.") @@ -3748,17 +3914,28 @@ class dblink(object): # get_owners is slow for large numbers of files, so # don't look them all up. collisions = collisions[:20] + + pkg_info_strs = {} self.lockdb() try: owners = self.vartree.dbapi._owners.get_owners(collisions) self.vartree.dbapi.flush_cache() + + for pkg in owners: + pkg = self.vartree.dbapi._pkg_str(pkg.mycpv, None) + pkg_info_str = "%s%s%s" % (pkg, + _slot_separator, pkg.slot) + if pkg.repo != _unknown_repo: + pkg_info_str += "%s%s" % (_repo_separator, + pkg.repo) + pkg_info_strs[pkg] = pkg_info_str + finally: self.unlockdb() for pkg, owned_files in owners.items(): - cpv = pkg.mycpv msg = [] - msg.append("%s" % cpv) + msg.append(pkg_info_strs[pkg.mycpv]) for f in sorted(owned_files): msg.append("\t%s" % os.path.join(destroot, f.lstrip(os.path.sep))) @@ -3814,6 +3991,20 @@ class dblink(object): self.delete() ensure_dirs(self.dbtmpdir) + downgrade = False + if self._installed_instance is not None and \ + vercmp(self.mycpv.version, + self._installed_instance.mycpv.version) < 0: + downgrade = True + + if self._installed_instance is not None: + rval = self._pre_merge_backup(self._installed_instance, downgrade) + if rval != os.EX_OK: + showMessage(_("!!! FAILED preinst: ") + + "quickpkg: %s\n" % rval, + level=logging.ERROR, noiselevel=-1) + return rval + # run preinst script showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \ {"cpv":self.mycpv, "destroot":destroot}) @@ -3835,32 +4026,26 @@ class dblink(object): # write local package counter for recording if counter is None: counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) - f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + with io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - f.write(_unicode_decode(str(counter))) - f.close() + errors='backslashreplace') as f: + f.write("%s" % counter) self.updateprotect() #if we have a file containing previously-merged config file md5sums, grab it. self.vartree.dbapi._fs_lock() try: + # Always behave like --noconfmem is enabled for downgrades + # so that people who don't know about this option are less + # likely to get confused when doing upgrade/downgrade cycles. cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - if "NOCONFMEM" in self.settings: + if "NOCONFMEM" in self.settings or downgrade: cfgfiledict["IGNORE"]=1 else: cfgfiledict["IGNORE"]=0 - # Always behave like --noconfmem is enabled for downgrades - # so that people who don't know about this option are less - # likely to get confused when doing upgrade/downgrade cycles. - for other in others_in_slot: - if vercmp(self.mycpv.version, other.mycpv.version) < 0: - cfgfiledict["IGNORE"] = 1 - break - rval = self._merge_contents(srcroot, destroot, cfgfiledict) if rval != os.EX_OK: return rval @@ -3970,6 +4155,7 @@ class dblink(object): try: self.delete() _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) + self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir)) finally: self.unlockdb() @@ -4014,9 +4200,9 @@ class dblink(object): self.vartree.dbapi.lock() try: try: - slot, counter = self.vartree.dbapi.aux_get( - cpv, ["SLOT", "COUNTER"]) - except KeyError: + slot = self.vartree.dbapi._pkg_str(cpv, None).slot + counter = self.vartree.dbapi.cpv_counter(cpv) + except (KeyError, InvalidData): pass else: has_vdb_entry = True @@ -4085,6 +4271,7 @@ class dblink(object): # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated. self._prune_plib_registry() + self._post_merge_sync() return os.EX_OK @@ -4100,7 +4287,7 @@ class dblink(object): x = -1 while True: x += 1 - backup_p = p + '.backup.' + str(x).rjust(4, '0') + backup_p = '%s.backup.%04d' % (p, x) try: os.lstat(backup_p) except OSError: @@ -4201,8 +4388,9 @@ class dblink(object): @type stufftomerge: String or List @param cfgfiledict: { File:mtime } mapping for config_protected files @type cfgfiledict: Dictionary - @param thismtime: The current time (typically long(time.time()) - @type thismtime: Long + @param thismtime: None or new mtime for merged files (expressed in seconds + in Python <3.3 and nanoseconds in Python >=3.3) + @type thismtime: None or Int @rtype: None or Boolean @return: 1. True on failure @@ -4227,18 +4415,18 @@ class dblink(object): # this is supposed to merge a list of files. There will be 2 forms of argument passing. if isinstance(stufftomerge, basestring): #A directory is specified. Figure out protection paths, listdir() it and process it. - mergelist = os.listdir(join(srcroot, stufftomerge)) - offset = stufftomerge + mergelist = [join(stufftomerge, child) for child in \ + os.listdir(join(srcroot, stufftomerge))] else: - mergelist = stufftomerge - offset = "" + mergelist = stufftomerge[:] - for i, x in enumerate(mergelist): + while mergelist: - mysrc = join(srcroot, offset, x) - mydest = join(destroot, offset, x) + relative_path = mergelist.pop() + mysrc = join(srcroot, relative_path) + mydest = join(destroot, relative_path) # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") - myrealdest = join(sep, offset, x) + myrealdest = join(sep, relative_path) # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] @@ -4333,9 +4521,26 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: + # Use lexists, since if the target happens to be a broken + # symlink then that should trigger an independent warning. + if not (os.path.lexists(myrealto) or + os.path.lexists(join(srcroot, myabsto))): + self._eqawarn('preinst', + [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") + % (relative_path, myabsto)]) + showMessage(">>> %s -> %s\n" % (mydest, myto)) - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) @@ -4429,11 +4634,17 @@ class dblink(object): os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) showMessage(">>> %s/\n" % mydest) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + outfile.write("dir "+myrealdest+"\n") # recurse and merge this directory - if self.mergeme(srcroot, destroot, outfile, secondhand, - join(offset, x), cfgfiledict, thismtime): - return 1 + mergelist.extend(join(relative_path, child) for child in + os.listdir(join(srcroot, relative_path))) + elif stat.S_ISREG(mymode): # we are merging a regular file mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) @@ -4489,7 +4700,10 @@ class dblink(object): cfgprot = cfgfiledict["IGNORE"] if not moveme: zing = "---" - mymtime = mystat[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + mymtime = mystat.st_mtime_ns + else: + mymtime = mystat[stat.ST_MTIME] else: moveme = 1 cfgprot = 1 @@ -4525,8 +4739,16 @@ class dblink(object): hardlink_candidates.append(mydest) zing = ">>>" + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node @@ -4537,6 +4759,12 @@ class dblink(object): sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) is not None: zing = ">>>" + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + else: return 1 if stat.S_ISFIFO(mymode): @@ -4545,6 +4773,52 @@ class dblink(object): outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") + def _merged_path(self, path, lstatobj, exists=True): + previous_path = self._device_path_map.get(lstatobj.st_dev) + if previous_path is None or previous_path is False or \ + (exists and len(path) < len(previous_path)): + if exists: + self._device_path_map[lstatobj.st_dev] = path + else: + # This entry is used to indicate that we've unmerged + # a file from this device, and later, this entry is + # replaced by a parent directory. + self._device_path_map[lstatobj.st_dev] = False + + def _post_merge_sync(self): + """ + Call this after merge or unmerge, in order to sync relevant files to + disk and avoid data-loss in the event of a power failure. This method + does nothing if FEATURES=merge-sync is disabled. + """ + if not self._device_path_map or \ + "merge-sync" not in self.settings.features: + return + + returncode = None + if platform.system() == "Linux": + + paths = [] + for path in self._device_path_map.values(): + if path is not False: + paths.append(path) + paths = tuple(paths) + + proc = SyncfsProcess(paths=paths, + scheduler=(self._scheduler or + portage._internal_caller and global_event_loop() or + EventLoop(main=False))) + proc.start() + returncode = proc.wait() + + if returncode is None or returncode != os.EX_OK: + try: + proc = subprocess.Popen(["sync"]) + except EnvironmentError: + pass + else: + proc.wait() + def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None, counter=None): """ @@ -4557,7 +4831,8 @@ class dblink(object): self.lockdb() self.vartree.dbapi._bump_mtime(self.mycpv) if self._scheduler is None: - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) try: retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes, @@ -4608,11 +4883,12 @@ class dblink(object): "returns contents of a file with whitespace converted to spaces" if not os.path.exists(self.dbdir+"/"+name): return "" - mydata = io.open( + with io.open( _unicode_encode(os.path.join(self.dbdir, name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read().split() + ) as f: + mydata = f.read().split() return " ".join(mydata) def copyfile(self,fname): @@ -4621,10 +4897,11 @@ class dblink(object): def getfile(self,fname): if not os.path.exists(self.dbdir+"/"+fname): return "" - return io.open(_unicode_encode(os.path.join(self.dbdir, fname), + with io.open(_unicode_encode(os.path.join(self.dbdir, fname), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read() + ) as f: + return f.read() def setfile(self,fname,data): kwargs = {} @@ -4633,16 +4910,18 @@ class dblink(object): else: kwargs['mode'] = 'w' kwargs['encoding'] = _encodings['repo.content'] - write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) + write_atomic(os.path.join(self.dbdir, fname), data, + **portage._native_kwargs(kwargs)) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] - mylines = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).readlines() + ) as f: + mylines = f.readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): @@ -4650,23 +4929,82 @@ class dblink(object): return myreturn def setelements(self,mylist,ename): - myelement = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - for x in mylist: - myelement.write(_unicode_decode(x+"\n")) - myelement.close() + errors='backslashreplace') as f: + for x in mylist: + f.write("%s\n" % x) def isregular(self): "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) + def _pre_merge_backup(self, backup_dblink, downgrade): + + if ("unmerge-backup" in self.settings.features or + (downgrade and "downgrade-backup" in self.settings.features)): + return self._quickpkg_dblink(backup_dblink, False, None) + + return os.EX_OK + + def _pre_unmerge_backup(self, background): + + if "unmerge-backup" in self.settings.features : + logfile = None + if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": + logfile = self.settings.get("PORTAGE_LOG_FILE") + return self._quickpkg_dblink(self, background, logfile) + + return os.EX_OK + + def _quickpkg_dblink(self, backup_dblink, background, logfile): + + trees = QueryCommand.get_db()[self.settings["EROOT"]] + bintree = trees["bintree"] + binpkg_path = bintree.getname(backup_dblink.mycpv) + if os.path.exists(binpkg_path) and \ + catsplit(backup_dblink.mycpv)[1] not in bintree.invalids: + return os.EX_OK + + self.lockdb() + try: + + if not backup_dblink.exists(): + # It got unmerged by a concurrent process. + return os.EX_OK + + # Call quickpkg for support of QUICKPKG_DEFAULT_OPTS and stuff. + quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], + "quickpkg") + + # Let quickpkg inherit the global vartree config's env. + env = dict(self.vartree.settings.items()) + env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" + + pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x] + if not pythonpath or \ + not os.path.samefile(pythonpath[0], portage._pym_path): + pythonpath.insert(0, portage._pym_path) + env['PYTHONPATH'] = ":".join(pythonpath) + + quickpkg_proc = SpawnProcess( + args=[portage._python_interpreter, quickpkg_binary, + "=%s" % (backup_dblink.mycpv,)], + background=background, env=env, + scheduler=self._scheduler, logfile=logfile) + quickpkg_proc.start() + + return quickpkg_proc.wait() + + finally: + self.unlockdb() + def merge(mycat, mypkg, pkgloc, infloc, myroot=None, settings=None, myebuild=None, mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, - scheduler=None): + scheduler=None, fd_pipes=None): """ @param myroot: ignored, settings['EROOT'] is used instead """ @@ -4681,10 +5019,12 @@ def merge(mycat, mypkg, pkgloc, infloc, merge_task = MergeProcess( mycat=mycat, mypkg=mypkg, settings=settings, treetype=mytree, vartree=vartree, - scheduler=(scheduler or PollScheduler().sched_iface), + scheduler=(scheduler or portage._internal_caller and + global_event_loop() or EventLoop(main=False)), background=background, blockers=blockers, pkgloc=pkgloc, infloc=infloc, myebuild=myebuild, mydbapi=mydbapi, - prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE')) + prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'), + fd_pipes=fd_pipes) merge_task.start() retcode = merge_task.wait() return retcode @@ -4864,13 +5204,11 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tar.addfile(tarinfo, f) f.close() else: - f = open(_unicode_encode(path, + with open(_unicode_encode(path, encoding=encoding, - errors='strict'), 'rb') - try: + errors='strict'), 'rb') as f: tar.addfile(tarinfo, f) - finally: - f.close() + else: tar.addfile(tarinfo) if onProgress: diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.pyo b/portage_with_autodep/pym/portage/dbapi/vartree.pyo Binary files differindex 7c186cf..745d15f 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/vartree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.py b/portage_with_autodep/pym/portage/dbapi/virtual.py index da15983..ba9745c 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.py +++ b/portage_with_autodep/pym/portage/dbapi/virtual.py @@ -1,6 +1,7 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand @@ -74,30 +75,55 @@ class fakedbapi(dbapi): @param metadata: dict """ self._clear_cache() - if not hasattr(mycpv, 'cp'): + + try: + mycp = mycpv.cp + except AttributeError: + mycp = None + try: + myslot = mycpv.slot + except AttributeError: + myslot = None + + if mycp is None or \ + (myslot is None and metadata is not None and metadata.get('SLOT')): if metadata is None: mycpv = _pkg_str(mycpv) else: - mycpv = _pkg_str(mycpv, slot=metadata.get('SLOT'), - repo=metadata.get('repository')) - mycp = mycpv.cp + mycpv = _pkg_str(mycpv, metadata=metadata, + settings=self.settings) + + mycp = mycpv.cp + try: + myslot = mycpv.slot + except AttributeError: + pass + self.cpvdict[mycpv] = metadata - myslot = None - if self._exclusive_slots and metadata: - myslot = metadata.get("SLOT", None) + if not self._exclusive_slots: + myslot = None if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT. for cpv in self.cpdict[mycp]: if mycpv != cpv: - other_metadata = self.cpvdict[cpv] - if other_metadata: - if myslot == other_metadata.get("SLOT", None): + try: + other_slot = cpv.slot + except AttributeError: + pass + else: + if myslot == other_slot: self.cpv_remove(cpv) break - if mycp not in self.cpdict: - self.cpdict[mycp] = [] - if not mycpv in self.cpdict[mycp]: - self.cpdict[mycp].append(mycpv) + + cp_list = self.cpdict.get(mycp) + if cp_list is None: + cp_list = [] + self.cpdict[mycp] = cp_list + try: + cp_list.remove(mycpv) + except ValueError: + pass + cp_list.append(mycpv) def cpv_remove(self,mycpv): """Removes a cpv from the list of available packages.""" diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.pyo b/portage_with_autodep/pym/portage/dbapi/virtual.pyo Binary files differindex 9f7c667..ca52263 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.pyo +++ b/portage_with_autodep/pym/portage/dbapi/virtual.pyo diff --git a/portage_with_autodep/pym/portage/debug.pyo b/portage_with_autodep/pym/portage/debug.pyo Binary files differindex 82a5e8f..a3c437d 100644 --- a/portage_with_autodep/pym/portage/debug.pyo +++ b/portage_with_autodep/pym/portage/debug.pyo diff --git a/portage_with_autodep/pym/portage/dep/__init__.py b/portage_with_autodep/pym/portage/dep/__init__.py index 152af0a..798903f 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.py +++ b/portage_with_autodep/pym/portage/dep/__init__.py @@ -1,7 +1,9 @@ # deps.py -- Portage dependency resolution functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'Atom', 'best_match_to_list', 'cpvequal', 'dep_getcpv', 'dep_getkey', 'dep_getslot', @@ -13,20 +15,6 @@ __all__ = [ '_repo_separator', '_slot_separator', ] -# DEPEND SYNTAX: -# -# 'use?' only affects the immediately following word! -# Nesting is the only legal way to form multiple '[!]use?' requirements. -# -# Where: 'a' and 'b' are use flags, and 'z' is a depend atom. -# -# "a? z" -- If 'a' in [use], then b is valid. -# "a? ( z )" -- Syntax with parenthesis. -# "a? b? z" -- Deprecated. -# "a? ( b? z )" -- Valid -# "a? ( b? ( z ) ) -- Valid -# - import re, sys import warnings from itertools import chain @@ -36,23 +24,164 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:cmp_sort_key,writemsg', ) -from portage import _unicode_decode -from portage.eapi import eapi_has_slot_deps, eapi_has_src_uri_arrows, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults, \ - eapi_has_repo_deps, eapi_allows_dots_in_PN, eapi_allows_dots_in_use_flags +from portage import _encodings, _unicode_decode, _unicode_encode +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidAtom, InvalidData, InvalidDependString from portage.localization import _ from portage.versions import catpkgsplit, catsplit, \ - vercmp, ververify, _cp, _cpv, _pkg_str, _unknown_repo + vercmp, ververify, _cp, _cpv, _pkg_str, _slot, _unknown_repo, _vr import portage.cache.mappings if sys.hexversion >= 0x3000000: basestring = str + _unicode = str +else: + _unicode = unicode + +# \w is [a-zA-Z0-9_] + +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot_separator = ":" +# loosly match SLOT, which may have an optional ABI part +_slot_loose = r'([\w+./*=-]+)' + +_use = r'\[.*\]' +_op = r'([=~]|[><]=?)' + +_repo_separator = "::" +_repo_name = r'[\w][\w-]*' +_repo_name_re = re.compile('^' + _repo_name + '$', re.UNICODE) +_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' + +_extended_cat = r'[\w+*][\w+.*-]*' + +_slot_dep_re_cache = {} + +def _get_slot_dep_re(eapi_attrs): + cache_key = eapi_attrs.slot_operator + slot_re = _slot_dep_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_operator: + slot_re = _slot + r'?(\*|=|/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE | re.UNICODE) + + _slot_dep_re_cache[cache_key] = slot_re + return slot_re + +def _match_slot(atom, pkg): + if pkg.slot == atom.slot: + if not atom.sub_slot: + return True + elif atom.sub_slot == pkg.sub_slot: + return True + return False + +_atom_re_cache = {} + +def _get_atom_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + cp_re = _cp['dots_allowed_in_PN'] + cpv_re = _cpv['dots_allowed_in_PN'] + else: + cp_re = _cp['dots_disallowed_in_PN'] + cpv_re = _cpv['dots_disallowed_in_PN'] + + atom_re = re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + cpv_re + ')|' + + '(?P<star>=' + cpv_re + r'\*)|' + + '(?P<simple>' + cp_re + '))' + + '(' + _slot_separator + _slot_loose + ')?' + + _repo + ')(' + _use + ')?$', re.VERBOSE | re.UNICODE) + + _atom_re_cache[cache_key] = atom_re + return atom_re + +_atom_wildcard_re_cache = {} + +def _get_atom_wildcard_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_wildcard_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + pkg_re = r'[\w+*][\w+.*-]*?' + else: + pkg_re = r'[\w+*][\w+*-]*?' + + atom_re = re.compile(r'((?P<simple>(' + + _extended_cat + r')/(' + pkg_re + r'(-' + _vr + ')?))' + \ + '|(?P<star>=((' + _extended_cat + r')/(' + pkg_re + r'))-(?P<version>\*\w+\*)))' + \ + '(:(?P<slot>' + _slot_loose + r'))?(' + + _repo_separator + r'(?P<repo>' + _repo_name + r'))?$', re.UNICODE) + + _atom_wildcard_re_cache[cache_key] = atom_re + return atom_re + +_usedep_re_cache = {} + +def _get_usedep_re(eapi_attrs): + """ + @param eapi_attrs: The EAPI attributes from _get_eapi_attrs + @type eapi_attrs: _eapi_attrs + @rtype: regular expression object + @return: A regular expression object that matches valid USE deps for the + given eapi. + """ + cache_key = eapi_attrs.dots_in_use_flags + usedep_re = _usedep_re_cache.get(cache_key) + if usedep_re is not None: + return usedep_re + + if eapi_attrs.dots_in_use_flags: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + usedep_re = re.compile(r'^(?P<prefix>[!-]?)(?P<flag>' + + _flag_re + r')(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$') + + _usedep_re_cache[cache_key] = usedep_re + return usedep_re -# Api consumers included in portage should set this to True. -# Once the relevant api changes are in a portage release with -# stable keywords, make these warnings unconditional. -_internal_warnings = False +_useflag_re_cache = {} + +def _get_useflag_re(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + @param eapi: The EAPI + @type eapi: String or None + @rtype: regular expression object + @return: A regular expression object that matches valid USE flags for the + given eapi. + """ + eapi_attrs = _get_eapi_attrs(eapi) + cache_key = eapi_attrs.dots_in_use_flags + useflag_re = _useflag_re_cache.get(cache_key) + if useflag_re is not None: + return useflag_re + + if eapi_attrs.dots_in_use_flags: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + useflag_re = re.compile(r'^' + flag_re + r'$') + + _useflag_re_cache[cache_key] = useflag_re + return useflag_re def cpvequal(cpv1, cpv2): """ @@ -109,7 +238,7 @@ def strip_empty(myarr): ('portage.dep.strip_empty',), DeprecationWarning, stacklevel=2) return [x for x in myarr if x] -def paren_reduce(mystr): +def paren_reduce(mystr, _deprecation_warn=True): """ Take a string and convert all paren enclosed entities into sublists and split the list elements by spaces. All redundant brackets are removed. @@ -123,7 +252,7 @@ def paren_reduce(mystr): @rtype: Array @return: The reduced string in an array """ - if _internal_warnings: + if portage._internal_caller and _deprecation_warn: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_reduce',), DeprecationWarning, stacklevel=2) mysplit = mystr.split() @@ -215,7 +344,7 @@ class paren_normalize(list): """Take a dependency structure as returned by paren_reduce or use_reduce and generate an equivalent structure that has no redundant lists.""" def __init__(self, src): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_normalize',), DeprecationWarning, stacklevel=2) list.__init__(self) @@ -250,7 +379,7 @@ class paren_normalize(list): self._zap_parens(x, dest) return dest -def paren_enclose(mylist, unevaluated_atom=False): +def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): """ Convert a list to a string with sublists enclosed with parens. @@ -267,7 +396,10 @@ def paren_enclose(mylist, unevaluated_atom=False): mystrparts = [] for x in mylist: if isinstance(x, list): - mystrparts.append("( "+paren_enclose(x)+" )") + if opconvert and x and x[0] == "||": + mystrparts.append("%s ( %s )" % (x[0], paren_enclose(x[1:]))) + else: + mystrparts.append("( %s )" % paren_enclose(x)) else: if unevaluated_atom: x = getattr(x, 'unevaluated_atom', x) @@ -308,7 +440,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i @return: The use reduced depend array """ if isinstance(depstr, list): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ "Pass the original dep string instead.") % \ ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) @@ -320,6 +452,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if matchall and matchnone: raise ValueError("portage.dep.use_reduce: 'matchall' and 'matchnone' are mutually exclusive") + eapi_attrs = _get_eapi_attrs(eapi) useflag_re = _get_useflag_re(eapi) def is_active(conditional): @@ -535,7 +668,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if not is_src_uri: raise InvalidDependString( _("SRC_URI arrow are only allowed in SRC_URI: token %s") % (pos+1,)) - if eapi is None or not eapi_has_src_uri_arrows(eapi): + if not eapi_attrs.src_uri_arrows: raise InvalidDependString( _("SRC_URI arrow not allowed in EAPI %s: token %s") % (eapi, pos+1)) need_simple_token = True @@ -608,7 +741,7 @@ def dep_opconvert(deplist): @return: The new list with the new ordering """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated. Use %s with the opconvert parameter set to True instead.") % \ ('portage.dep.dep_opconvert', 'portage.dep.use_reduce'), DeprecationWarning, stacklevel=2) @@ -639,7 +772,7 @@ def flatten(mylist): @rtype: List @return: A single list containing only non-list elements. """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.flatten',), DeprecationWarning, stacklevel=2) @@ -651,30 +784,9 @@ def flatten(mylist): newlist.append(x) return newlist - -_usedep_re = { - "dots_disallowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), - "dots_allowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), -} - -def _get_usedep_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE deps for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _usedep_re["dots_allowed_in_use_flags"] - else: - return _usedep_re["dots_disallowed_in_use_flags"] - class _use_dep(object): - __slots__ = ("__weakref__", "eapi", "conditional", "missing_enabled", "missing_disabled", + __slots__ = ("_eapi_attrs", "conditional", "missing_enabled", "missing_disabled", "disabled", "enabled", "tokens", "required") class _conditionals_class(object): @@ -700,10 +812,10 @@ class _use_dep(object): 'not_equal': '!%s=', } - def __init__(self, use, eapi, enabled_flags=None, disabled_flags=None, missing_enabled=None, \ + def __init__(self, use, eapi_attrs, enabled_flags=None, disabled_flags=None, missing_enabled=None, missing_disabled=None, conditional=None, required=None): - self.eapi = eapi + self._eapi_attrs = eapi_attrs if enabled_flags is not None: #A shortcut for the classe's own methods. @@ -732,7 +844,7 @@ class _use_dep(object): no_default = set() conditional = {} - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in use: m = usedep_re.match(x) @@ -800,6 +912,14 @@ class _use_dep(object): return "" return "[%s]" % (",".join(self.tokens),) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __repr__(self): return "portage.dep._use_dep(%s)" % repr(self.tokens) @@ -835,7 +955,7 @@ class _use_dep(object): disabled_flags = set(self.disabled) tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -871,7 +991,7 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, required=self.required) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -893,7 +1013,7 @@ class _use_dep(object): def validate_flag(flag): return is_valid_flag(flag) or flag in all_defaults - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -985,7 +1105,7 @@ class _use_dep(object): tokens.append(x) conditional.setdefault("disabled", set()).add(flag) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, \ conditional=conditional, required=self.required) @@ -1005,7 +1125,7 @@ class _use_dep(object): missing_disabled = self.missing_disabled tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -1041,15 +1161,10 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=missing_enabled, missing_disabled=missing_disabled, required=self.required) -if sys.hexversion < 0x3000000: - _atom_base = unicode -else: - _atom_base = str - -class Atom(_atom_base): +class Atom(_unicode): """ For compatibility with existing atom string manipulation code, this @@ -1068,51 +1183,72 @@ class Atom(_atom_base): def __init__(self, forbid_overlap=False): self.overlap = self._overlap(forbid=forbid_overlap) - def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): - return _atom_base.__new__(cls, s) + return _unicode.__new__(cls, s) - def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): if isinstance(s, Atom): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ - (_atom_base, type(s))) + (_unicode, type(s))) - if not isinstance(s, _atom_base): - # Avoid TypeError from _atom_base.__init__ with PyPy. + if not isinstance(s, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. s = _unicode_decode(s) - _atom_base.__init__(s) + _unicode.__init__(s) + + eapi_attrs = _get_eapi_attrs(eapi) + atom_re = _get_atom_re(eapi_attrs) - atom_re = _get_atom_re(eapi) - if eapi_has_repo_deps(eapi): - allow_repo = True + self.__dict__['eapi'] = eapi + if eapi is not None: + # Ignore allow_repo when eapi is specified. + allow_repo = eapi_attrs.repo_deps + else: + if allow_repo is None: + allow_repo = True + blocker_prefix = "" if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) if blocker.overlap.forbid: + blocker_prefix = s[:2] s = s[2:] else: + blocker_prefix = s[:1] s = s[1:] else: blocker = False self.__dict__['blocker'] = blocker m = atom_re.match(s) extended_syntax = False + extended_version = None if m is None: if allow_wildcard: - m = _get_atom_wildcard_re(eapi).match(s) + atom_re = _get_atom_wildcard_re(eapi_attrs) + m = atom_re.match(s) if m is None: raise InvalidAtom(self) - op = None gdict = m.groupdict() - cpv = cp = gdict['simple'] + if m.group('star') is not None: + op = '=*' + base = atom_re.groupindex['star'] + cp = m.group(base + 1) + cpv = m.group('star')[1:] + extended_version = m.group(base + 4) + else: + op = None + cpv = cp = m.group('simple') + if m.group(atom_re.groupindex['simple'] + 3) is not None: + raise InvalidAtom(self) if cpv.find("**") != -1: raise InvalidAtom(self) - slot = gdict['slot'] - repo = gdict['repo'] + slot = m.group('slot') + repo = m.group('repo') use_str = None extended_syntax = True else: @@ -1155,9 +1291,38 @@ class Atom(_atom_base): except InvalidData: # plain cp, wildcard, or something self.__dict__['cpv'] = cpv - self.__dict__['version'] = None + self.__dict__['version'] = extended_version self.__dict__['repo'] = repo - self.__dict__['slot'] = slot + if slot is None: + self.__dict__['slot'] = None + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None + else: + slot_re = _get_slot_dep_re(eapi_attrs) + slot_match = slot_re.match(slot) + if slot_match is None: + raise InvalidAtom(self) + if eapi_attrs.slot_operator: + self.__dict__['slot'] = slot_match.group(1) + sub_slot = slot_match.group(2) + if sub_slot is not None: + sub_slot = sub_slot.lstrip("/") + if sub_slot in ("*", "="): + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = sub_slot + else: + slot_operator = None + if sub_slot is not None and sub_slot[-1:] == "=": + slot_operator = sub_slot[-1:] + sub_slot = sub_slot[:-1] + self.__dict__['sub_slot'] = sub_slot + self.__dict__['slot_operator'] = slot_operator + if self.slot is not None and self.slot_operator == "*": + raise InvalidAtom(self) + else: + self.__dict__['slot'] = slot + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None self.__dict__['operator'] = op self.__dict__['extended_syntax'] = extended_syntax @@ -1168,16 +1333,19 @@ class Atom(_atom_base): if _use is not None: use = _use else: - use = _use_dep(use_str[1:-1].split(","), eapi) - without_use = Atom(m.group('without_use'), allow_repo=allow_repo) + use = _use_dep(use_str[1:-1].split(","), eapi_attrs) + without_use = Atom(blocker_prefix + m.group('without_use'), + allow_repo=allow_repo) else: use = None if unevaluated_atom is not None and \ unevaluated_atom.use is not None: # unevaluated_atom.use is used for IUSE checks when matching # packages, so it must not propagate to without_use - without_use = Atom(s, allow_wildcard=allow_wildcard, - allow_repo=allow_repo) + without_use = Atom(_unicode(self), + allow_wildcard=allow_wildcard, + allow_repo=allow_repo, + eapi=eapi) else: without_use = self @@ -1193,16 +1361,16 @@ class Atom(_atom_base): if not isinstance(eapi, basestring): raise TypeError('expected eapi argument of ' + \ '%s, got %s: %s' % (basestring, type(eapi), eapi,)) - if self.slot and not eapi_has_slot_deps(eapi): + if self.slot and not eapi_attrs.slot_deps: raise InvalidAtom( _("Slot deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') if self.use: - if not eapi_has_use_deps(eapi): + if not eapi_attrs.use_deps: raise InvalidAtom( _("Use deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') - elif not eapi_has_use_dep_defaults(eapi) and \ + elif not eapi_attrs.use_dep_defaults and \ (self.use.missing_enabled or self.use.missing_disabled): raise InvalidAtom( _("Use dep defaults are not allowed in EAPI %s: '%s'") \ @@ -1225,12 +1393,21 @@ class Atom(_atom_base): "conditional '%s' in atom '%s' is not in IUSE") \ % (flag, conditional_str % flag, self) raise InvalidAtom(msg, category='IUSE.missing') - if self.blocker and self.blocker.overlap.forbid and not eapi_has_strong_blocks(eapi): + if self.blocker and self.blocker.overlap.forbid and not eapi_attrs.strong_blocks: raise InvalidAtom( _("Strong blocks are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') @property + def slot_operator_built(self): + """ + Returns True if slot_operator == "=" and sub_slot is not None. + NOTE: foo/bar:2= is unbuilt and returns False, whereas foo/bar:2/2= + is built and returns True. + """ + return self.slot_operator == "=" and self.sub_slot is not None + + @property def without_repo(self): if self.repo is None: return self @@ -1239,18 +1416,29 @@ class Atom(_atom_base): @property def without_slot(self): - if self.slot is None: + if self.slot is None and self.slot_operator is None: return self - return Atom(self.replace(_slot_separator + self.slot, '', 1), + atom = remove_slot(self) + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += _unicode(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) def with_repo(self, repo): atom = remove_slot(self) - if self.slot is not None: - atom += _slot_separator + self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator atom += _repo_separator + repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def with_slot(self, slot): @@ -1258,7 +1446,7 @@ class Atom(_atom_base): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def __setattr__(self, name, value): @@ -1307,10 +1495,16 @@ class Atom(_atom_base): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.evaluate_conditionals(use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -1329,20 +1523,32 @@ class Atom(_atom_base): if not self.use: return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.violated_conditionals(other_use, is_valid_flag, parent_use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def _eval_qa_conditionals(self, use_mask, use_force): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use._eval_qa_conditionals(use_mask, use_force) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def __copy__(self): @@ -1366,7 +1572,7 @@ def extended_cp_match(extended_cp, other_cp): extended_cp_re = _extended_cp_re_cache.get(extended_cp) if extended_cp_re is None: extended_cp_re = re.compile("^" + re.escape(extended_cp).replace( - r'\*', '[^/]*') + "$") + r'\*', '[^/]*') + "$", re.UNICODE) _extended_cp_re_cache[extended_cp] = extended_cp_re return extended_cp_re.match(other_cp) is not None @@ -1644,77 +1850,8 @@ def dep_getusedeps( depend ): open_bracket = depend.find( '[', open_bracket+1 ) return tuple(use_list) -# \w is [a-zA-Z0-9_] - -# 2.1.3 A slot name may contain any of the characters [A-Za-z0-9+_.-]. -# It must not begin with a hyphen or a dot. -_slot_separator = ":" -_slot = r'([\w+][\w+.-]*)' -_slot_re = re.compile('^' + _slot + '$', re.VERBOSE) - -_use = r'\[.*\]' -_op = r'([=~]|[><]=?)' -_repo_separator = "::" -_repo_name = r'[\w][\w-]*' -_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' - -_atom_re = { - "dots_disallowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_disallowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_disallowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_disallowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), - "dots_allowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_allowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_allowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_allowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), -} - -def _get_atom_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_re["dots_allowed_in_PN"] - else: - return _atom_re["dots_disallowed_in_PN"] - -_extended_cat = r'[\w+*][\w+.*-]*' -_extended_pkg = { - "dots_disallowed_in_PN": r'[\w+*][\w+*-]*?', - "dots_allowed_in_PN": r'[\w+*][\w+.*-]*?', -} - -_atom_wildcard_re = { - "dots_disallowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_disallowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), - "dots_allowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_allowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), -} - -def _get_atom_wildcard_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_wildcard_re["dots_allowed_in_PN"] - else: - return _atom_wildcard_re["dots_disallowed_in_PN"] - -_useflag_re = { - "dots_disallowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), - "dots_allowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), -} - -def _get_useflag_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE flags for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _useflag_re["dots_allowed_in_use_flags"] - else: - return _useflag_re["dots_disallowed_in_use_flags"] - -def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=False): +def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, + allow_repo=False, eapi=None): """ Check to see if a depend atom is valid @@ -1731,9 +1868,15 @@ def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=Fal 1) False if the atom is invalid 2) True if the atom is valid """ + + if eapi is not None and isinstance(atom, Atom) and atom.eapi != eapi: + # We'll construct a new atom with the given eapi. + atom = _unicode(atom) + try: if not isinstance(atom, Atom): - atom = Atom(atom, allow_wildcard=allow_wildcard, allow_repo=allow_repo) + atom = Atom(atom, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) if not allow_blockers and atom.blocker: return False return True @@ -1859,19 +2002,23 @@ def best_match_to_list(mypkg, mylist): """ operator_values = {'=':6, '~':5, '=*':4, '>':2, '<':2, '>=':2, '<=':2, None:1} - maxvalue = -2 + maxvalue = -99 bestm = None mypkg_cpv = None for x in match_to_list(mypkg, mylist): if x.extended_syntax: - if dep_getslot(x) is not None: + if x.operator == '=*': if maxvalue < 0: maxvalue = 0 bestm = x - else: + elif x.slot is not None: if maxvalue < -1: maxvalue = -1 bestm = x + else: + if maxvalue < -2: + maxvalue = -2 + bestm = x continue if dep_getslot(x) is not None: if maxvalue < 3: @@ -1955,7 +2102,8 @@ def match_from_list(mydep, candidate_list): mylist = [] - if operator is None: + if mydep.extended_syntax: + for x in candidate_list: cp = getattr(x, "cp", None) if cp is None: @@ -1966,8 +2114,38 @@ def match_from_list(mydep, candidate_list): if cp is None: continue - if cp == mycpv or (mydep.extended_syntax and \ - extended_cp_match(mydep.cp, cp)): + if cp == mycpv or extended_cp_match(mydep.cp, cp): + mylist.append(x) + + if mylist and mydep.operator == "=*": + + candidate_list = mylist + mylist = [] + # Currently, only \*\w+\* is supported. + ver = mydep.version[1:-1] + + for x in candidate_list: + x_ver = getattr(x, "version", None) + if x_ver is None: + xs = catpkgsplit(remove_slot(x)) + if xs is None: + continue + x_ver = "-".join(xs[-2:]) + if ver in x_ver: + mylist.append(x) + + elif operator is None: + for x in candidate_list: + cp = getattr(x, "cp", None) + if cp is None: + mysplit = catpkgsplit(remove_slot(x)) + if mysplit is not None: + cp = mysplit[0] + '/' + mysplit[1] + + if cp is None: + continue + + if cp == mydep.cp: mylist.append(x) elif operator == "=": # Exact match @@ -1983,19 +2161,40 @@ def match_from_list(mydep, candidate_list): # XXX: Nasty special casing for leading zeros # Required as =* is a literal prefix match, so can't # use vercmp - mysplit = catpkgsplit(mycpv) - myver = mysplit[2].lstrip("0") + myver = mycpv_cps[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - mycpv_cmp = mysplit[0]+"/"+mysplit[1]+"-"+myver + if myver == mycpv_cps[2]: + mycpv_cmp = mycpv + else: + # Use replace to preserve the revision part if it exists + # (mycpv_cps[3] can't be trusted because in contains r0 + # even when the input has no revision part). + mycpv_cmp = mycpv.replace( + mydep.cp + "-" + mycpv_cps[2], + mydep.cp + "-" + myver, 1) for x in candidate_list: - xs = getattr(x, "cpv_split", None) - if xs is None: - xs = catpkgsplit(remove_slot(x)) + try: + x.cp + except AttributeError: + try: + pkg = _pkg_str(remove_slot(x)) + except InvalidData: + continue + else: + pkg = x + + xs = pkg.cpv_split myver = xs[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - xcpv = xs[0]+"/"+xs[1]+"-"+myver + if myver == xs[2]: + xcpv = pkg.cpv + else: + # Use replace to preserve the revision part if it exists. + xcpv = pkg.cpv.replace( + pkg.cp + "-" + xs[2], + pkg.cp + "-" + myver, 1) if xcpv.startswith(mycpv_cmp): mylist.append(x) @@ -2048,16 +2247,33 @@ def match_from_list(mydep, candidate_list): else: raise KeyError(_("Unknown operator: %s") % mydep) - if slot is not None and not mydep.extended_syntax: + if mydep.slot is not None: candidate_list = mylist mylist = [] for x in candidate_list: - xslot = getattr(x, "slot", False) - if xslot is False: + x_pkg = None + try: + x.cpv + except AttributeError: xslot = dep_getslot(x) - if xslot is not None and xslot != slot: - continue - mylist.append(x) + if xslot is not None: + try: + x_pkg = _pkg_str(remove_slot(x), slot=xslot) + except InvalidData: + continue + else: + x_pkg = x + + if x_pkg is None: + mylist.append(x) + else: + try: + x_pkg.slot + except AttributeError: + mylist.append(x) + else: + if _match_slot(mydep, x_pkg): + mylist.append(x) if mydep.unevaluated_atom.use: candidate_list = mylist @@ -2071,26 +2287,26 @@ def match_from_list(mydep, candidate_list): continue if mydep.use: - - missing_enabled = mydep.use.missing_enabled.difference(x.iuse.all) - missing_disabled = mydep.use.missing_disabled.difference(x.iuse.all) + is_valid_flag = x.iuse.is_valid_flag + missing_enabled = frozenset(flag for flag in + mydep.use.missing_enabled if not is_valid_flag(flag)) + missing_disabled = frozenset(flag for flag in + mydep.use.missing_disabled if not is_valid_flag(flag)) if mydep.use.enabled: - if mydep.use.enabled.intersection(missing_disabled): + if any(f in mydep.use.enabled for f in missing_disabled): continue need_enabled = mydep.use.enabled.difference(use.enabled) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(f not in missing_enabled for f in need_enabled): continue if mydep.use.disabled: - if mydep.use.disabled.intersection(missing_enabled): + if any(f in mydep.use.disabled for f in missing_enabled): continue need_disabled = mydep.use.disabled.intersection(use.enabled) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(f not in missing_disabled for f in need_disabled): continue mylist.append(x) @@ -2110,9 +2326,9 @@ def match_from_list(mydep, candidate_list): return mylist def human_readable_required_use(required_use): - return required_use.replace("^^", "exactly-one-of").replace("||", "any-of") + return required_use.replace("^^", "exactly-one-of").replace("||", "any-of").replace("??", "at-most-one-of") -def get_required_use_flags(required_use): +def get_required_use_flags(required_use, eapi=None): """ Returns a set of use flags that are used in the given REQUIRED_USE string @@ -2122,6 +2338,12 @@ def get_required_use_flags(required_use): @return: Set of use flags that are used in the given REQUIRED_USE string """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + mysplit = required_use.split() level = 0 stack = [[]] @@ -2150,7 +2372,7 @@ def get_required_use_flags(required_use): l = stack.pop() ignore = False if stack[level]: - if stack[level][-1] in ("||", "^^") or \ + if stack[level][-1] in valid_operators or \ (not isinstance(stack[level][-1], bool) and \ stack[level][-1][-1] == "?"): ignore = True @@ -2162,15 +2384,14 @@ def get_required_use_flags(required_use): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) need_bracket = True stack[level].append(token) else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2225,7 +2446,7 @@ class _RequiredUseBranch(object): complex_nesting = False node = self while node != None and not complex_nesting: - if node._operator in ("||", "^^"): + if node._operator in ("||", "^^", "??"): complex_nesting = True else: node = node._parent @@ -2246,7 +2467,7 @@ class _RequiredUseBranch(object): if sys.hexversion < 0x3000000: __nonzero__ = __bool__ -def check_required_use(required_use, use, iuse_match): +def check_required_use(required_use, use, iuse_match, eapi=None): """ Checks if the use flags listed in 'use' satisfy all constraints specified in 'constraints'. @@ -2262,6 +2483,12 @@ def check_required_use(required_use, use, iuse_match): @return: Indicates if REQUIRED_USE constraints are satisfied """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + def is_active(token): if token.startswith("!"): flag = token[1:] @@ -2271,6 +2498,11 @@ def check_required_use(required_use, use, iuse_match): is_negated = False if not flag or not iuse_match(flag): + if not eapi_attrs.required_use_at_most_one_of and flag == "?": + msg = _("Operator '??' is not supported with EAPI '%s'") \ + % (eapi,) + e = InvalidData(msg, category='EAPI.incompatible') + raise InvalidDependString(msg, errors=(e,)) msg = _("USE flag '%s' is not in IUSE") \ % (flag,) e = InvalidData(msg, category='IUSE.missing') @@ -2288,6 +2520,8 @@ def check_required_use(required_use, use, iuse_match): return (True in argument) elif operator == "^^": return (argument.count(True) == 1) + elif operator == "??": + return (argument.count(True) <= 1) elif operator[-1] == "?": return (False not in argument) @@ -2317,7 +2551,7 @@ def check_required_use(required_use, use, iuse_match): l = stack.pop() op = None if stack[level]: - if stack[level][-1] in ("||", "^^"): + if stack[level][-1] in valid_operators: op = stack[level].pop() satisfied = is_satisfied(op, l) stack[level].append(satisfied) @@ -2346,7 +2580,7 @@ def check_required_use(required_use, use, iuse_match): stack[level].append(satisfied) if len(node._children) <= 1 or \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2362,7 +2596,7 @@ def check_required_use(required_use, use, iuse_match): raise AssertionError( "node is not last child of parent") - elif len(node._children) == 1 and op in ("||", "^^"): + elif len(node._children) == 1 and op in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2372,7 +2606,7 @@ def check_required_use(required_use, use, iuse_match): node._children[0]._parent = node._parent node = node._children[0] if node._operator is None and \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2386,7 +2620,7 @@ def check_required_use(required_use, use, iuse_match): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2396,8 +2630,7 @@ def check_required_use(required_use, use, iuse_match): node._children.append(child) node = child else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2425,16 +2658,16 @@ def extract_affecting_use(mystr, atom, eapi=None): that decide if the given atom is in effect. Example usage: - >>> extract_use_cond('sasl? ( dev-libs/cyrus-sasl ) \ + >>> extract_affecting_use('sasl? ( dev-libs/cyrus-sasl ) \ !minimal? ( cxx? ( dev-libs/cyrus-sasl ) )', 'dev-libs/cyrus-sasl') - (['sasl', 'minimal', 'cxx']) + {'cxx', 'minimal', 'sasl'} - @param dep: The dependency string + @param mystr: The dependency string @type mystr: String @param atom: The atom to get into effect @type atom: String - @rtype: Tuple of two lists of strings - @return: List of use flags that need to be enabled, List of use flag that need to be disabled + @rtype: Set of strings + @return: Set of use flags affecting given atom """ useflag_re = _get_useflag_re(eapi) mysplit = mystr.split() @@ -2540,3 +2773,48 @@ def extract_affecting_use(mystr, atom, eapi=None): _("malformed syntax: '%s'") % mystr) return affecting_use + +def extract_unpack_dependencies(src_uri, unpackers): + """ + Return unpack dependencies string for given SRC_URI string. + + @param src_uri: SRC_URI string + @type src_uri: String + @param unpackers: Dictionary mapping archive suffixes to dependency strings + @type unpackers: Dictionary + @rtype: String + @return: Dependency string specifying packages required to unpack archives. + """ + src_uri = src_uri.split() + + depend = [] + for i in range(len(src_uri)): + if src_uri[i][-1] == "?" or src_uri[i] in ("(", ")"): + depend.append(src_uri[i]) + elif (i+1 < len(src_uri) and src_uri[i+1] == "->") or src_uri[i] == "->": + continue + else: + for suffix in sorted(unpackers, key=lambda x: len(x), reverse=True): + suffix = suffix.lower() + if src_uri[i].lower().endswith(suffix): + depend.append(unpackers[suffix]) + break + + while True: + cleaned_depend = depend[:] + for i in range(len(cleaned_depend)): + if cleaned_depend[i] is None: + continue + elif cleaned_depend[i] == "(" and cleaned_depend[i+1] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + elif cleaned_depend[i][-1] == "?" and cleaned_depend[i+1] == "(" and cleaned_depend[i+2] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + cleaned_depend[i+2] = None + if depend == cleaned_depend: + break + else: + depend = [x for x in cleaned_depend if x is not None] + + return " ".join(depend) diff --git a/portage_with_autodep/pym/portage/dep/__init__.pyo b/portage_with_autodep/pym/portage/dep/__init__.pyo Binary files differindex c78bb23..515ec01 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.pyo +++ b/portage_with_autodep/pym/portage/dep/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dep/dep_check.py b/portage_with_autodep/pym/portage/dep/dep_check.py index 99a5eb0..b5ace3d 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.py +++ b/portage_with_autodep/pym/portage/dep/dep_check.py @@ -1,16 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] import logging +import operator import portage -from portage import _unicode_decode from portage.dep import Atom, match_from_list, use_reduce from portage.exception import InvalidDependString, ParseError from portage.localization import _ from portage.util import writemsg, writemsg_level +from portage.util.SlotObject import SlotObject from portage.versions import vercmp, _pkg_str def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", @@ -160,7 +163,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", # According to GLEP 37, RDEPEND is the only dependency # type that is valid for new-style virtuals. Repoman # should enforce this. - depstring = pkg.metadata['RDEPEND'] + depstring = pkg._metadata['RDEPEND'] pkg_kwargs = kwargs.copy() pkg_kwargs["myuse"] = pkg_use_enabled(pkg) if edebug: @@ -183,7 +186,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", del mytrees["virt_parent"] if not mycheck[0]: - raise ParseError(_unicode_decode("%s: %s '%s'") % \ + raise ParseError("%s: %s '%s'" % \ (pkg, mycheck[1], depstring)) # pull in the new-style virtual @@ -254,6 +257,10 @@ def dep_eval(deplist): return 0 return 1 +class _dep_choice(SlotObject): + __slots__ = ('atoms', 'slot_map', 'cp_map', 'all_available', + 'all_installed_slots') + def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): """ Takes an unreduced and reduced deplist and removes satisfied dependencies. @@ -316,6 +323,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): priority = trees[myroot].get("priority") graph_db = trees[myroot].get("graph_db") graph = trees[myroot].get("graph") + want_update_pkg = trees[myroot].get("want_update_pkg") vardb = None if "vartree" in trees[myroot]: vardb = trees[myroot]["vartree"].dbapi @@ -324,6 +332,13 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): else: mydbapi = trees[myroot]["porttree"].dbapi + try: + mydbapi_match_pkgs = mydbapi.match_pkgs + except AttributeError: + def mydbapi_match_pkgs(atom): + return [mydbapi._pkg_str(cpv, atom.repo) + for cpv in mydbapi.match(atom)] + # Sort the deps into installed, not installed but already # in the graph and other, not installed and not in the graph # and other, with values of [[required_atom], availablility] @@ -347,24 +362,17 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): continue # Ignore USE dependencies here since we don't want USE # settings to adversely affect || preference evaluation. - avail_pkg = mydbapi.match(atom.without_use) + avail_pkg = mydbapi_match_pkgs(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, eapi=eapi, - slot=slot, repo=repo) - avail_slot = Atom("%s:%s" % (atom.cp, slot)) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) if not avail_pkg: all_available = False all_use_satisfied = False break if atom.use: - avail_pkg_use = mydbapi.match(atom) + avail_pkg_use = mydbapi_match_pkgs(atom) if not avail_pkg_use: all_use_satisfied = False else: @@ -372,13 +380,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg_use = avail_pkg_use[-1] if avail_pkg_use != avail_pkg: avail_pkg = avail_pkg_use - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, - eapi=eapi, slot=slot, repo=repo) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) slot_map[avail_slot] = avail_pkg highest_cpv = cp_map.get(avail_pkg.cp) @@ -386,7 +388,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): vercmp(avail_pkg.version, highest_cpv.version) > 0: cp_map[avail_pkg.cp] = avail_pkg - this_choice = (atoms, slot_map, cp_map, all_available) + this_choice = _dep_choice(atoms=atoms, slot_map=slot_map, + cp_map=cp_map, all_available=all_available, + all_installed_slots=False) if all_available: # The "all installed" criterion is not version or slot specific. # If any version of a package is already in the graph then we @@ -407,6 +411,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): not slot_atom.startswith("virtual/"): all_installed_slots = False break + this_choice.all_installed_slots = all_installed_slots if graph_db is None: if all_use_satisfied: if all_installed: @@ -468,8 +473,27 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): elif all_installed: if all_installed_slots: preferred_installed.append(this_choice) - else: + elif parent is None or want_update_pkg is None: preferred_any_slot.append(this_choice) + else: + # When appropriate, prefer a slot that is not + # installed yet for bug #478188. + want_update = True + for slot_atom, avail_pkg in slot_map.items(): + if avail_pkg in graph: + continue + # New-style virtuals have zero cost to install. + if slot_atom.startswith("virtual/") or \ + vardb.match(slot_atom): + continue + if not want_update_pkg(parent, avail_pkg): + want_update = False + break + + if want_update: + preferred_installed.append(this_choice) + else: + preferred_any_slot.append(this_choice) else: preferred_non_installed.append(this_choice) else: @@ -490,6 +514,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): all_installed = False if all_installed: + this_choice.all_installed_slots = True other_installed.append(this_choice) elif some_installed: other_installed_some.append(this_choice) @@ -506,22 +531,23 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for choices in choice_bins: if len(choices) < 2: continue + # Prefer choices with all_installed_slots for bug #480736. + choices.sort(key=operator.attrgetter('all_installed_slots'), + reverse=True) for choice_1 in choices[1:]: - atoms_1, slot_map_1, cp_map_1, all_available_1 = choice_1 - cps = set(cp_map_1) + cps = set(choice_1.cp_map) for choice_2 in choices: if choice_1 is choice_2: # choice_1 will not be promoted, so move on break - atoms_2, slot_map_2, cp_map_2, all_available_2 = choice_2 - intersecting_cps = cps.intersection(cp_map_2) + intersecting_cps = cps.intersection(choice_2.cp_map) if not intersecting_cps: continue has_upgrade = False has_downgrade = False for cp in intersecting_cps: - version_1 = cp_map_1[cp] - version_2 = cp_map_2[cp] + version_1 = choice_1.cp_map[cp] + version_2 = choice_2.cp_map[cp] difference = vercmp(version_1.version, version_2.version) if difference != 0: if difference > 0: @@ -538,9 +564,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for allow_masked in (False, True): for choices in choice_bins: - for atoms, slot_map, cp_map, all_available in choices: - if all_available or allow_masked: - return atoms + for choice in choices: + if choice.all_available or allow_masked: + return choice.atoms assert(False) # This point should not be reachable @@ -575,18 +601,15 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, mymasks = set() useforce = set() - useforce.add(mysettings["ARCH"]) if use == "all": - # This masking/forcing is only for repoman. In other cases, relevant - # masking/forcing should have already been applied via - # config.regenerate(). Also, binary or installed packages may have - # been built with flags that are now masked, and it would be - # inconsistent to mask them now. Additionally, myuse may consist of - # flags from a parent package that is being merged to a $ROOT that is - # different from the one that mysettings represents. + # This is only for repoman, in order to constrain the use_reduce + # matchall behavior to account for profile use.mask/force. The + # ARCH/archlist code here may be redundant, since the profile + # really should be handling ARCH masking/forcing itself. mymasks.update(mysettings.usemask) mymasks.update(mysettings.archlist()) mymasks.discard(mysettings["ARCH"]) + useforce.add(mysettings["ARCH"]) useforce.update(mysettings.useforce) useforce.difference_update(mymasks) @@ -609,14 +632,17 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, # dependencies so that things like --depclean work as well as possible # in spite of partial invalidity. if not current_parent.installed: - eapi = current_parent.metadata['EAPI'] + eapi = current_parent.eapi - try: - mysplit = use_reduce(depstring, uselist=myusesplit, masklist=mymasks, \ - matchall=(use=="all"), excludeall=useforce, opconvert=True, \ - token_class=Atom, eapi=eapi) - except InvalidDependString as e: - return [0, _unicode_decode("%s") % (e,)] + if isinstance(depstring, list): + mysplit = depstring + else: + try: + mysplit = use_reduce(depstring, uselist=myusesplit, + masklist=mymasks, matchall=(use=="all"), excludeall=useforce, + opconvert=True, token_class=Atom, eapi=eapi) + except InvalidDependString as e: + return [0, "%s" % (e,)] if mysplit == []: #dependencies were reduced to nothing @@ -630,10 +656,10 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, use_force=useforce, use_mask=mymasks, use_cache=use_cache, use_binaries=use_binaries, myroot=myroot, trees=trees) except ParseError as e: - return [0, _unicode_decode("%s") % (e,)] + return [0, "%s" % (e,)] - mysplit2=mysplit[:] - mysplit2=dep_wordreduce(mysplit2,mysettings,mydbapi,mode,use_cache=use_cache) + mysplit2 = dep_wordreduce(mysplit, + mysettings, mydbapi, mode, use_cache=use_cache) if mysplit2 is None: return [0, _("Invalid token")] diff --git a/portage_with_autodep/pym/portage/dep/dep_check.pyo b/portage_with_autodep/pym/portage/dep/dep_check.pyo Binary files differindex 1b9e03f..feec00e 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.pyo +++ b/portage_with_autodep/pym/portage/dep/dep_check.pyo diff --git a/portage_with_autodep/pym/portage/dispatch_conf.py b/portage_with_autodep/pym/portage/dispatch_conf.py index 4c68dfc..79daa9f 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.py +++ b/portage_with_autodep/pym/portage/dispatch_conf.py @@ -1,5 +1,5 @@ # archive_conf.py -- functionality common to archive-conf and dispatch-conf -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -31,9 +31,17 @@ def diffstatusoutput(cmd, file1, file2): # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [portage._unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) @@ -43,7 +51,7 @@ def diffstatusoutput(cmd, file1, file2): return (proc.wait(), output) def read_config(mandatory_opts): - eprefix = portage.const.EPREFIX + eprefix = portage.settings["EPREFIX"] config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf") loader = KeyValuePairFileLoader(config_path, None) opts, errors = loader.load() diff --git a/portage_with_autodep/pym/portage/dispatch_conf.pyo b/portage_with_autodep/pym/portage/dispatch_conf.pyo Binary files differindex 6239859..2241d20 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.pyo +++ b/portage_with_autodep/pym/portage/dispatch_conf.pyo diff --git a/portage_with_autodep/pym/portage/eapi.py b/portage_with_autodep/pym/portage/eapi.py index 79cf891..4f77910 100644 --- a/portage_with_autodep/pym/portage/eapi.py +++ b/portage_with_autodep/pym/portage/eapi.py @@ -1,12 +1,22 @@ # Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import collections + +from portage import eapi_is_supported + def eapi_has_iuse_defaults(eapi): return eapi != "0" +def eapi_has_iuse_effective(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_slot_deps(eapi): return eapi != "0" +def eapi_has_slot_operator(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python") + def eapi_has_src_uri_arrows(eapi): return eapi not in ("0", "1") @@ -34,8 +44,11 @@ def eapi_exports_merge_type(eapi): def eapi_exports_replace_vars(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_exports_EBUILD_PHASE_FUNC(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_exports_REPOSITORY(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_has_pkg_pretend(eapi): return eapi not in ("0", "1", "2", "3") @@ -49,14 +62,83 @@ def eapi_has_dosed_dohard(eapi): def eapi_has_required_use(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_has_required_use_at_most_one_of(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_use_dep_defaults(eapi): return eapi not in ("0", "1", "2", "3") def eapi_has_repo_deps(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_PN(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_use_flags(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") + +def eapi_supports_stable_use_forcing_and_masking(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + +def eapi_allows_directories_on_profile_level_and_repository_level(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_use_aliases(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_automatic_unpack_dependencies(eapi): + return eapi in ("5-progress",) + +def eapi_has_hdepend(eapi): + return eapi in ("5-hdepend",) + +def eapi_has_targetroot(eapi): + return eapi in ("5-hdepend",) + +_eapi_attrs = collections.namedtuple('_eapi_attrs', + 'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC ' + 'feature_flag_test feature_flag_targetroot ' + 'hdepend iuse_defaults iuse_effective ' + 'repo_deps required_use required_use_at_most_one_of slot_operator slot_deps ' + 'src_uri_arrows strong_blocks use_deps use_dep_defaults') + +_eapi_attrs_cache = {} + +def _get_eapi_attrs(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + An unsupported eapi is handled the same as when eapi is None, which may + be helpful for handling of corrupt EAPI metadata in essential functions + such as pkgsplit. + """ + eapi_attrs = _eapi_attrs_cache.get(eapi) + if eapi_attrs is not None: + return eapi_attrs + + orig_eapi = eapi + if eapi is not None and not eapi_is_supported(eapi): + eapi = None + + eapi_attrs = _eapi_attrs( + dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)), + dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags(eapi)), + exports_EBUILD_PHASE_FUNC = (eapi is None or eapi_exports_EBUILD_PHASE_FUNC(eapi)), + feature_flag_test = True, + feature_flag_targetroot = (eapi is not None and eapi_has_targetroot(eapi)), + hdepend = (eapi is not None and eapi_has_hdepend(eapi)), + iuse_defaults = (eapi is None or eapi_has_iuse_defaults(eapi)), + iuse_effective = (eapi is not None and eapi_has_iuse_effective(eapi)), + repo_deps = (eapi is None or eapi_has_repo_deps(eapi)), + required_use = (eapi is None or eapi_has_required_use(eapi)), + required_use_at_most_one_of = (eapi is None or eapi_has_required_use_at_most_one_of(eapi)), + slot_deps = (eapi is None or eapi_has_slot_deps(eapi)), + slot_operator = (eapi is None or eapi_has_slot_operator(eapi)), + src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)), + strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)), + use_deps = (eapi is None or eapi_has_use_deps(eapi)), + use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults(eapi)) + ) + + _eapi_attrs_cache[orig_eapi] = eapi_attrs + return eapi_attrs diff --git a/portage_with_autodep/pym/portage/eapi.pyo b/portage_with_autodep/pym/portage/eapi.pyo Binary files differindex ce67ab1..cfce6bd 100644 --- a/portage_with_autodep/pym/portage/eapi.pyo +++ b/portage_with_autodep/pym/portage/eapi.pyo diff --git a/portage_with_autodep/pym/portage/eclass_cache.py b/portage_with_autodep/pym/portage/eclass_cache.py index cb2cf8a..8c209c8 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.py +++ b/portage_with_autodep/pym/portage/eclass_cache.py @@ -1,7 +1,9 @@ -# Copyright 2005-2011 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + __all__ = ["cache"] import stat @@ -12,6 +14,7 @@ import errno from portage.exception import FileNotFound, PermissionDenied from portage import os from portage import checksum +from portage import _shell_quote if sys.hexversion >= 0x3000000: long = int @@ -60,6 +63,7 @@ class cache(object): self.eclasses = {} # {"Name": hashed_path} self._eclass_locations = {} + self._eclass_locations_str = None # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. # ~harring @@ -98,6 +102,7 @@ class cache(object): self.porttrees = self.porttrees + other.porttrees self.eclasses.update(other.eclasses) self._eclass_locations.update(other._eclass_locations) + self._eclass_locations_str = None def update_eclasses(self): self.eclasses = {} @@ -169,3 +174,10 @@ class cache(object): ec_dict[x] = self.eclasses[x] return ec_dict + + @property + def eclass_locations_string(self): + if self._eclass_locations_str is None: + self._eclass_locations_str = " ".join(_shell_quote(x) + for x in reversed(self.porttrees)) + return self._eclass_locations_str diff --git a/portage_with_autodep/pym/portage/eclass_cache.pyo b/portage_with_autodep/pym/portage/eclass_cache.pyo Binary files differindex ebe3463..ffd56f7 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.pyo +++ b/portage_with_autodep/pym/portage/eclass_cache.pyo diff --git a/portage_with_autodep/pym/portage/elog/__init__.pyo b/portage_with_autodep/pym/portage/elog/__init__.pyo Binary files differindex 39dc595..6bf9898 100644 --- a/portage_with_autodep/pym/portage/elog/__init__.pyo +++ b/portage_with_autodep/pym/portage/elog/__init__.pyo diff --git a/portage_with_autodep/pym/portage/elog/filtering.pyo b/portage_with_autodep/pym/portage/elog/filtering.pyo Binary files differindex 3a040cc..c67c7bb 100644 --- a/portage_with_autodep/pym/portage/elog/filtering.pyo +++ b/portage_with_autodep/pym/portage/elog/filtering.pyo diff --git a/portage_with_autodep/pym/portage/elog/messages.pyo b/portage_with_autodep/pym/portage/elog/messages.pyo Binary files differindex c033f55..0e3239f 100644 --- a/portage_with_autodep/pym/portage/elog/messages.pyo +++ b/portage_with_autodep/pym/portage/elog/messages.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_custom.pyo b/portage_with_autodep/pym/portage/elog/mod_custom.pyo Binary files differindex 317fab4..cc95c24 100644 --- a/portage_with_autodep/pym/portage/elog/mod_custom.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_custom.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_echo.pyo b/portage_with_autodep/pym/portage/elog/mod_echo.pyo Binary files differindex 6a00d4c..be383e2 100644 --- a/portage_with_autodep/pym/portage/elog/mod_echo.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_echo.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail.pyo b/portage_with_autodep/pym/portage/elog/mod_mail.pyo Binary files differindex 5d87aa6..bfed1d3 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo Binary files differindex d7306b5..17c2135 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save.py b/portage_with_autodep/pym/portage/elog/mod_save.py index c69f4a3..7b1cd46 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.py +++ b/portage_with_autodep/pym/portage/elog/mod_save.py @@ -1,7 +1,8 @@ # elog/mod_save.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import errno import io import time import portage @@ -47,11 +48,22 @@ def process(mysettings, key, logentries, fulltext): elogfilename = os.path.join(log_subdir, cat + ':' + elogfilename) _ensure_log_subdirs(logdir, log_subdir) - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['content'], errors='backslashreplace') - elogfile.write(_unicode_decode(fulltext)) - elogfile.close() + try: + with io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), mode='w', + encoding=_encodings['content'], + errors='backslashreplace') as elogfile: + elogfile.write(_unicode_decode(fulltext)) + except IOError as e: + func_call = "open('%s', 'w')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(log_subdir) diff --git a/portage_with_autodep/pym/portage/elog/mod_save.pyo b/portage_with_autodep/pym/portage/elog/mod_save.pyo Binary files differindex fb28b76..82d683d 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.py b/portage_with_autodep/pym/portage/elog/mod_save_summary.py index 347f66e..786f894 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.py +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.py @@ -1,8 +1,12 @@ # elog/mod_save_summary.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import errno import io +import sys import time import portage from portage import os @@ -37,9 +41,21 @@ def process(mysettings, key, logentries, fulltext): # TODO: Locking elogfilename = elogdir+"/summary.log" - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='backslashreplace') + try: + elogfile = io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], + errors='backslashreplace') + except IOError as e: + func_call = "open('%s', 'a')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(elogdir) @@ -58,17 +74,19 @@ def process(mysettings, key, logentries, fulltext): apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0) - time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z", - time.localtime(time.time())) - # Avoid potential UnicodeDecodeError later. + time_fmt = "%Y-%m-%d %H:%M:%S %Z" + if sys.hexversion < 0x3000000: + time_fmt = _unicode_encode(time_fmt) + time_str = time.strftime(time_fmt, time.localtime(time.time())) + # Avoid potential UnicodeDecodeError in Python 2, since strftime + # returns bytes in Python 2, and %Z may contain non-ascii chars. time_str = _unicode_decode(time_str, encoding=_encodings['content'], errors='replace') - elogfile.write(_unicode_decode( - _(">>> Messages generated by process " + + elogfile.write(_(">>> Messages generated by process " "%(pid)d on %(time)s for package %(pkg)s:\n\n") % - {"pid": os.getpid(), "time": time_str, "pkg": key})) + {"pid": os.getpid(), "time": time_str, "pkg": key}) elogfile.write(_unicode_decode(fulltext)) - elogfile.write(_unicode_decode("\n")) + elogfile.write("\n") elogfile.close() return elogfilename diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo Binary files differindex 8f99c51..19e4422 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo Binary files differindex c7b4248..bb01208 100644 --- a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo diff --git a/portage_with_autodep/pym/portage/env/__init__.pyo b/portage_with_autodep/pym/portage/env/__init__.pyo Binary files differindex 846aea3..b45054e 100644 --- a/portage_with_autodep/pym/portage/env/__init__.pyo +++ b/portage_with_autodep/pym/portage/env/__init__.pyo diff --git a/portage_with_autodep/pym/portage/env/config.pyo b/portage_with_autodep/pym/portage/env/config.pyo Binary files differindex 13c2e86..0a7faa3 100644 --- a/portage_with_autodep/pym/portage/env/config.pyo +++ b/portage_with_autodep/pym/portage/env/config.pyo diff --git a/portage_with_autodep/pym/portage/env/loaders.py b/portage_with_autodep/pym/portage/env/loaders.py index 372bc12..f869884 100644 --- a/portage_with_autodep/pym/portage/env/loaders.py +++ b/portage_with_autodep/pym/portage/env/loaders.py @@ -1,10 +1,14 @@ # config.py -- Portage Config -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno import io import stat +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:writemsg', +) from portage import os from portage import _encodings from portage import _unicode_decode @@ -149,17 +153,21 @@ class FileLoader(DataLoader): func = self.lineParser for fn in RecursiveFileLoader(self.fname): try: - f = io.open(_unicode_encode(fn, + with io.open(_unicode_encode(fn, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace') + encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() except EnvironmentError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): + if e.errno == errno.EACCES: + writemsg(_("Permission denied: '%s'\n") % fn, noiselevel=-1) + del e + elif e.errno in (errno.ENOENT, errno.ESTALE): + del e + else: raise - del e - continue - for line_num, line in enumerate(f): - func(line, line_num, data, errors) - f.close() + else: + for line_num, line in enumerate(lines): + func(line, line_num, data, errors) return (data, errors) def lineParser(self, line, line_num, data, errors): diff --git a/portage_with_autodep/pym/portage/env/loaders.pyo b/portage_with_autodep/pym/portage/env/loaders.pyo Binary files differindex 2622a9f..91c3efa 100644 --- a/portage_with_autodep/pym/portage/env/loaders.pyo +++ b/portage_with_autodep/pym/portage/env/loaders.pyo diff --git a/portage_with_autodep/pym/portage/env/validators.pyo b/portage_with_autodep/pym/portage/env/validators.pyo Binary files differindex cd18adb..24e1510 100644 --- a/portage_with_autodep/pym/portage/env/validators.pyo +++ b/portage_with_autodep/pym/portage/env/validators.pyo diff --git a/portage_with_autodep/pym/portage/exception.py b/portage_with_autodep/pym/portage/exception.py index 5ccd750..1388c49 100644 --- a/portage_with_autodep/pym/portage/exception.py +++ b/portage_with_autodep/pym/portage/exception.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import signal @@ -11,26 +11,35 @@ if sys.hexversion >= 0x3000000: class PortageException(Exception): """General superclass for portage exceptions""" - def __init__(self,value): - self.value = value[:] - if isinstance(self.value, basestring): - self.value = _unicode_decode(self.value, - encoding=_encodings['content'], errors='replace') + if sys.hexversion >= 0x3000000: + def __init__(self,value): + self.value = value[:] - def __str__(self): - if isinstance(self.value, basestring): - return self.value - else: - return _unicode_decode(repr(self.value), - encoding=_encodings['content'], errors='replace') - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ + def __str__(self): + if isinstance(self.value, str): + return self.value + else: + return repr(self.value) + else: + def __init__(self,value): + self.value = value[:] + if isinstance(self.value, basestring): + self.value = _unicode_decode(self.value, + encoding=_encodings['content'], errors='replace') + + def __unicode__(self): + if isinstance(self.value, unicode): + return self.value + else: + return _unicode_decode(repr(self.value), + encoding=_encodings['content'], errors='replace') def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') + if isinstance(self.value, unicode): + return _unicode_encode(self.value, + encoding=_encodings['content'], errors='backslashreplace') + else: + return repr(self.value) class CorruptionError(PortageException): """Corruption indication""" diff --git a/portage_with_autodep/pym/portage/exception.pyo b/portage_with_autodep/pym/portage/exception.pyo Binary files differindex 3a60e7c..c60c3a0 100644 --- a/portage_with_autodep/pym/portage/exception.pyo +++ b/portage_with_autodep/pym/portage/exception.pyo diff --git a/portage_with_autodep/pym/portage/getbinpkg.py b/portage_with_autodep/pym/portage/getbinpkg.py index 212f788..ff656ba 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.py +++ b/portage_with_autodep/pym/portage/getbinpkg.py @@ -1,7 +1,9 @@ # getbinpkg.py -- Portage binary-package helper functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + from portage.output import colorize from portage.cache.mappings import slot_dict_class from portage.localization import _ @@ -18,6 +20,7 @@ import socket import time import tempfile import base64 +import warnings _all_errors = [NotImplementedError, ValueError, socket.error] @@ -65,6 +68,10 @@ if sys.hexversion >= 0x3000000: long = int def make_metadata_dict(data): + + warnings.warn("portage.getbinpkg.make_metadata_dict() is deprecated", + DeprecationWarning, stacklevel=2) + myid,myglob = data mydict = {} @@ -84,6 +91,10 @@ class ParseLinks(html_parser_HTMLParser): """Parser class that overrides HTMLParser to grab all anchors from an html page and provide suffix and prefix limitors""" def __init__(self): + + warnings.warn("portage.getbinpkg.ParseLinks is deprecated", + DeprecationWarning, stacklevel=2) + self.PL_anchors = [] html_parser_HTMLParser.__init__(self) @@ -122,6 +133,9 @@ def create_conn(baseurl,conn=None): optional connection. If connection is already active, it is passed on. baseurl is reduced to address and is returned in tuple (conn,address)""" + warnings.warn("portage.getbinpkg.create_conn() is deprecated", + DeprecationWarning, stacklevel=2) + parts = baseurl.split("://",1) if len(parts) != 2: raise ValueError(_("Provided URI does not " @@ -221,6 +235,10 @@ def create_conn(baseurl,conn=None): def make_ftp_request(conn, address, rest=None, dest=None): """(conn,address,rest) --- uses the conn object to request the data from address and issuing a rest if it is passed.""" + + warnings.warn("portage.getbinpkg.make_ftp_request() is deprecated", + DeprecationWarning, stacklevel=2) + try: if dest: @@ -270,6 +288,9 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): the data from address, performing Location forwarding and using the optional params and headers.""" + warnings.warn("portage.getbinpkg.make_http_request() is deprecated", + DeprecationWarning, stacklevel=2) + rc = 0 response = None while (rc == 0) or (rc == 301) or (rc == 302): @@ -312,6 +333,10 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0): + + warnings.warn("portage.getbinpkg.match_in_array() is deprecated", + DeprecationWarning, stacklevel=2) + myarray = [] if not (prefix and suffix): @@ -352,6 +377,9 @@ def dir_get_list(baseurl,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.dir_get_list() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -400,6 +428,9 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -446,30 +477,53 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): return myid -def file_get(baseurl,dest,conn=None,fcmd=None,filename=None): +def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None, + fcmd_vars=None): """(baseurl,dest,fcmd=) -- Takes a base url to connect to and read from. URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>""" if not fcmd: + + warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd " + "parameter is deprecated", DeprecationWarning, stacklevel=2) + return file_get_lib(baseurl,dest,conn) - if not filename: - filename = os.path.basename(baseurl) - variables = { - "DISTDIR": dest, - "URI": baseurl, - "FILE": filename - } + variables = {} + + if fcmd_vars is not None: + variables.update(fcmd_vars) + + if "DISTDIR" not in variables: + if dest is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "DISTDIR")) + variables["DISTDIR"] = dest + + if "URI" not in variables: + if baseurl is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "URI")) + variables["URI"] = baseurl + + if "FILE" not in variables: + if filename is None: + filename = os.path.basename(variables["URI"]) + variables["FILE"] = filename from portage.util import varexpand from portage.process import spawn myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stdout.fileno() + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stdout__.fileno() } + sys.__stdout__.flush() + sys.__stderr__.flush() retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes) if retval != os.EX_OK: sys.stderr.write(_("Fetcher exited with a failure condition.\n")) @@ -481,6 +535,9 @@ def file_get_lib(baseurl,dest,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_lib() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -524,6 +581,10 @@ def file_get_lib(baseurl,dest,conn=None): def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None): """(baseurl,conn,chunk_size,verbose) -- """ + + warnings.warn("portage.getbinpkg.dir_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -557,7 +618,9 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= out.write(_("Loaded metadata pickle.\n")) out.flush() metadatafile.close() - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError): + except (SystemExit, KeyboardInterrupt): + raise + except Exception: metadata = {} if baseurl not in metadata: metadata[baseurl]={} diff --git a/portage_with_autodep/pym/portage/getbinpkg.pyo b/portage_with_autodep/pym/portage/getbinpkg.pyo Binary files differindex 53ec2e9..578acd9 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.pyo +++ b/portage_with_autodep/pym/portage/getbinpkg.pyo diff --git a/portage_with_autodep/pym/portage/glsa.py b/portage_with_autodep/pym/portage/glsa.py index 1857695..cac0f1a 100644 --- a/portage_with_autodep/pym/portage/glsa.py +++ b/portage_with_autodep/pym/portage/glsa.py @@ -1,7 +1,7 @@ # Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import io import sys @@ -9,8 +9,12 @@ try: from urllib.request import urlopen as urllib_request_urlopen except ImportError: from urllib import urlopen as urllib_request_urlopen +import codecs import re +import operator import xml.dom.minidom +from io import StringIO +from functools import reduce import portage from portage import os @@ -19,13 +23,13 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.versions import pkgsplit, vercmp, best from portage.util import grabfile -from portage.const import CACHE_PATH +from portage.const import PRIVATE_PATH from portage.localization import _ from portage.dep import _slot_separator # Note: the space for rgt and rlt is important !! # FIXME: use slot deps instead, requires GLSA format versioning -opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", +opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", "rge": ">=~", "rle": "<=~", "rgt": " >~", "rlt": " <~"} NEWLINE_ESCAPE = "!;\\n" # some random string to mark newlines that should be preserved SPACE_ESCAPE = "!;_" # some random string to mark spaces that should be preserved @@ -39,22 +43,22 @@ def get_applied_glsas(settings): @rtype: list @return: list of glsa IDs """ - return grabfile(os.path.join(settings["EROOT"], CACHE_PATH, "glsa")) + return grabfile(os.path.join(settings["EROOT"], PRIVATE_PATH, "glsa_injected")) # TODO: use the textwrap module instead def wrap(text, width, caption=""): """ Wraps the given text at column I{width}, optionally indenting - it so that no text is under I{caption}. It's possible to encode + it so that no text is under I{caption}. It's possible to encode hard linebreaks in I{text} with L{NEWLINE_ESCAPE}. - + @type text: String @param text: the text to be wrapped @type width: Integer @param width: the column at which the text should be wrapped @type caption: String - @param caption: this string is inserted at the beginning of the + @param caption: this string is inserted at the beginning of the return value and the paragraph is indented up to C{len(caption)}. @rtype: String @@ -65,7 +69,7 @@ def wrap(text, width, caption=""): text = text.replace(2*NEWLINE_ESCAPE, NEWLINE_ESCAPE+" "+NEWLINE_ESCAPE) words = text.split() indentLevel = len(caption)+1 - + for w in words: if line != "" and line[-1] == "\n": rValue += line @@ -94,10 +98,10 @@ def get_glsa_list(myconfig): Returns a list of all available GLSAs in the given repository by comparing the filelist there with the pattern described in the config. - + @type myconfig: portage.config @param myconfig: Portage settings instance - + @rtype: List of Strings @return: a list of GLSA IDs in this repository """ @@ -113,10 +117,10 @@ def get_glsa_list(myconfig): dirlist = os.listdir(repository) prefix = "glsa-" suffix = ".xml" - + for f in dirlist: try: - if f[:len(prefix)] == prefix: + if f[:len(prefix)] == prefix and f[-1*len(suffix):] == suffix: rValue.append(f[len(prefix):-1*len(suffix)]) except IndexError: pass @@ -125,22 +129,20 @@ def get_glsa_list(myconfig): def getListElements(listnode): """ Get all <li> elements for a given <ol> or <ul> node. - + @type listnode: xml.dom.Node @param listnode: <ul> or <ol> list to get the elements for @rtype: List of Strings @return: a list that contains the value of the <li> elements """ - rValue = [] if not listnode.nodeName in ["ul", "ol"]: raise GlsaFormatException("Invalid function call: listnode is not <ul> or <ol>") - for li in listnode.childNodes: - if li.nodeType != xml.dom.Node.ELEMENT_NODE: - continue - rValue.append(getText(li, format="strip")) + rValue = [getText(li, format="strip") \ + for li in listnode.childNodes \ + if li.nodeType == xml.dom.Node.ELEMENT_NODE] return rValue -def getText(node, format): +def getText(node, format, textfd = None): """ This is the main parser function. It takes a node and traverses recursive over the subnodes, getting the text of each (and the @@ -148,7 +150,7 @@ def getText(node, format): parameter the text might be formatted by adding/removing newlines, tabs and spaces. This function is only useful for the GLSA DTD, it's not applicable for other DTDs. - + @type node: xml.dom.Node @param node: the root node to start with the parsing @type format: String @@ -158,45 +160,54 @@ def getText(node, format): replaces multiple spaces with one space. I{xml} does some more formatting, depending on the type of the encountered nodes. + @type textfd: writable file-like object + @param textfd: the file-like object to write the output to @rtype: String @return: the (formatted) content of the node and its subnodes + except if textfd was not none """ - rValue = "" + if not textfd: + textfd = StringIO() + returnNone = False + else: + returnNone = True if format in ["strip", "keep"]: if node.nodeName in ["uri", "mail"]: - rValue += node.childNodes[0].data+": "+node.getAttribute("link") + textfd.write(node.childNodes[0].data+": "+node.getAttribute("link")) else: for subnode in node.childNodes: if subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: - rValue += getText(subnode, format) - else: + getText(subnode, format, textfd) + else: # format = "xml" for subnode in node.childNodes: if subnode.nodeName == "p": for p_subnode in subnode.childNodes: if p_subnode.nodeName == "#text": - rValue += p_subnode.data.strip() + textfd.write(p_subnode.data.strip()) elif p_subnode.nodeName in ["uri", "mail"]: - rValue += p_subnode.childNodes[0].data - rValue += " ( "+p_subnode.getAttribute("link")+" )" - rValue += NEWLINE_ESCAPE + textfd.write(p_subnode.childNodes[0].data) + textfd.write(" ( "+p_subnode.getAttribute("link")+" )") + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "ul": for li in getListElements(subnode): - rValue += "-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write("-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "ol": i = 0 for li in getListElements(subnode): i = i+1 - rValue += str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write(str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "code": - rValue += getText(subnode, format="keep").replace("\n", NEWLINE_ESCAPE) - if rValue[-1*len(NEWLINE_ESCAPE):] != NEWLINE_ESCAPE: - rValue += NEWLINE_ESCAPE + textfd.write(getText(subnode, format="keep").lstrip().replace("\n", NEWLINE_ESCAPE)) + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: raise GlsaFormatException(_("Invalid Tag found: "), subnode.nodeName) + if returnNone: + return None + rValue = textfd.getvalue() if format == "strip": rValue = rValue.strip(" \n\t") rValue = re.sub("[\s]{2,}", " ", rValue) @@ -206,7 +217,7 @@ def getMultiTagsText(rootnode, tagname, format): """ Returns a list with the text of all subnodes of type I{tagname} under I{rootnode} (which itself is not parsed) using the given I{format}. - + @type rootnode: xml.dom.Node @param rootnode: the node to search for I{tagname} @type tagname: String @@ -216,16 +227,15 @@ def getMultiTagsText(rootnode, tagname, format): @rtype: List of Strings @return: a list containing the text of all I{tagname} childnodes """ - rValue = [] - for e in rootnode.getElementsByTagName(tagname): - rValue.append(getText(e, format)) + rValue = [getText(e, format) \ + for e in rootnode.getElementsByTagName(tagname)] return rValue def makeAtom(pkgname, versionNode): """ - creates from the given package name and information in the + creates from the given package name and information in the I{versionNode} a (syntactical) valid portage atom. - + @type pkgname: String @param pkgname: the name of the package for this atom @type versionNode: xml.dom.Node @@ -248,9 +258,9 @@ def makeAtom(pkgname, versionNode): def makeVersion(versionNode): """ - creates from the information in the I{versionNode} a + creates from the information in the I{versionNode} a version string (format <op><version>). - + @type versionNode: xml.dom.Node @param versionNode: a <vulnerable> or <unaffected> Node that contains the version information for this atom @@ -270,17 +280,17 @@ def makeVersion(versionNode): def match(atom, dbapi, match_type="default"): """ - wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on + wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on the given atom. - + @type atom: string @param atom: a <~ or >~ atom or a normal portage atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to dbapi.xmatch + @param match_type: if != "default" passed as first argument to dbapi.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -296,15 +306,15 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave as > and < except that they are limited to the same version, the range only applies to the revision part. - + @type revisionAtom: string @param revisionAtom: a <~ or >~ atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to portdb.xmatch + @param match_type: if != "default" passed as first argument to portdb.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -325,18 +335,19 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): if eval(r1+" "+revisionAtom[0:2]+" "+r2): rValue.append(v) return rValue - + def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize=True): """ Checks if the systemstate is matching an atom in I{vulnerableList} and returns string describing - the lowest version for the package that matches an atom in + the lowest version for the package that matches an atom in I{unaffectedList} and is greater than the currently installed - version or None if the system is not affected. Both - I{vulnerableList} and I{unaffectedList} should have the + version. It will return an empty list if the system is affected, + and no upgrade is possible or None if the system is not affected. + Both I{vulnerableList} and I{unaffectedList} should have the same base package. - + @type vulnerableList: List of Strings @param vulnerableList: atoms matching vulnerable package versions @type unaffectedList: List of Strings @@ -347,46 +358,51 @@ def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize= @param vardbapi: Installed package repository @type minimize: Boolean @param minimize: True for a least-change upgrade, False for emerge-like algorithm - + @rtype: String | None @return: the lowest unaffected version that is greater than the installed version. - """ - rValue = None - v_installed = [] - u_installed = [] - for v in vulnerableList: - v_installed += match(v, vardbapi) + """ + rValue = "" + v_installed = reduce(operator.add, [match(v, vardbapi) for v in vulnerableList], []) + u_installed = reduce(operator.add, [match(u, vardbapi) for u in unaffectedList], []) - for u in unaffectedList: - u_installed += match(u, vardbapi) - - install_unaffected = True - for i in v_installed: - if i not in u_installed: - install_unaffected = False + # remove all unaffected atoms from vulnerable list + v_installed = list(set(v_installed).difference(set(u_installed))) - if install_unaffected: - return rValue - + if not v_installed: + return None + + # this tuple holds all vulnerable atoms, and the related upgrade atom + vuln_update = [] + avail_updates = set() for u in unaffectedList: - mylist = match(u, portdbapi, match_type="match-all") - for c in mylist: - i = best(v_installed) - if vercmp(c.version, i.version) > 0 \ - and (rValue == None \ - or not match("="+rValue, portdbapi) \ - or (minimize ^ (vercmp(c.version, rValue.version) > 0)) \ - and match("="+c, portdbapi)) \ - and portdbapi.aux_get(c, ["SLOT"]) == vardbapi.aux_get(best(v_installed), ["SLOT"]): - rValue = c - return rValue + # TODO: This had match_type="match-all" before. I don't think it should + # since we disregarded masked items later anyway (match(=rValue, "porttree")) + avail_updates.update(match(u, portdbapi)) + # if an atom is already installed, we should not consider it for upgrades + avail_updates.difference_update(u_installed) + + for vuln in v_installed: + update = "" + for c in avail_updates: + c_pv = portage.catpkgsplit(c) + if vercmp(c.version, vuln.version) > 0 \ + and (update == "" \ + or (minimize ^ (vercmp(c.version, update.version) > 0))) \ + and portdbapi._pkg_str(c, None).slot == vardbapi._pkg_str(vuln, None).slot: + update = c_pv[0]+"/"+c_pv[1]+"-"+c_pv[2] + if c_pv[3] != "r0": # we don't like -r0 for display + update += "-"+c_pv[3] + vuln_update.append([vuln, update]) + + return vuln_update def format_date(datestr): """ Takes a date (announced, revised) date from a GLSA and formats it as readable text (i.e. "January 1, 2008"). - + @type date: String @param date: the date string to reformat @rtype: String @@ -396,16 +412,16 @@ def format_date(datestr): splitdate = datestr.split("-", 2) if len(splitdate) != 3: return datestr - + # This cannot raise an error as we use () instead of [] splitdate = (int(x) for x in splitdate) - + from datetime import date try: d = date(*splitdate) except ValueError: return datestr - + # TODO We could format to local date format '%x' here? return _unicode_decode(d.strftime("%B %d, %Y"), encoding=_encodings['content'], errors='replace') @@ -417,7 +433,7 @@ class GlsaTypeException(Exception): class GlsaFormatException(Exception): pass - + class GlsaArgumentException(Exception): pass @@ -429,9 +445,9 @@ class Glsa: """ def __init__(self, myid, myconfig, vardbapi, portdbapi): """ - Simple constructor to set the ID, store the config and gets the + Simple constructor to set the ID, store the config and gets the XML data by calling C{self.read()}. - + @type myid: String @param myid: String describing the id for the GLSA object (standard GLSAs have an ID of the form YYYYMM-nn) or an existing @@ -461,7 +477,7 @@ class Glsa: """ Here we build the filename from the config and the ID and pass it to urllib to fetch it from the filesystem or a remote server. - + @rtype: None @return: None """ @@ -473,15 +489,21 @@ class Glsa: myurl = "file://"+self.nr else: myurl = repository + "glsa-%s.xml" % str(self.nr) - self.parse(urllib_request_urlopen(myurl)) + + f = urllib_request_urlopen(myurl) + try: + self.parse(f) + finally: + f.close() + return None def parse(self, myfile): """ - This method parses the XML file and sets up the internal data + This method parses the XML file and sets up the internal data structures by calling the different helper functions in this module. - + @type myfile: String @param myfile: Filename to grab the XML data from @rtype: None @@ -504,27 +526,27 @@ class Glsa: self.title = getText(myroot.getElementsByTagName("title")[0], format="strip") self.synopsis = getText(myroot.getElementsByTagName("synopsis")[0], format="strip") self.announced = format_date(getText(myroot.getElementsByTagName("announced")[0], format="strip")) - - count = 1 + # Support both formats of revised: # <revised>December 30, 2007: 02</revised> # <revised count="2">2007-12-30</revised> revisedEl = myroot.getElementsByTagName("revised")[0] self.revised = getText(revisedEl, format="strip") - if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or - (sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))): - count = revisedEl.getAttribute("count") - elif (self.revised.find(":") >= 0): - (self.revised, count) = self.revised.split(":") - + count = revisedEl.attributes.get("count") + if count is None: + if self.revised.find(":") >= 0: + (self.revised, count) = self.revised.split(":") + else: + count = 1 + self.revised = format_date(self.revised) - + try: self.count = int(count) except ValueError: # TODO should this raise a GlsaFormatException? self.count = 1 - + # now the optional and 0-n toplevel, #PCDATA tags and references try: self.access = getText(myroot.getElementsByTagName("access")[0], format="strip") @@ -532,7 +554,7 @@ class Glsa: self.access = "" self.bugs = getMultiTagsText(myroot, "bug", format="strip") self.references = getMultiTagsText(myroot.getElementsByTagName("references")[0], "uri", format="keep") - + # and now the formatted text elements self.description = getText(myroot.getElementsByTagName("description")[0], format="xml") self.workaround = getText(myroot.getElementsByTagName("workaround")[0], format="xml") @@ -542,7 +564,7 @@ class Glsa: try: self.background = getText(myroot.getElementsByTagName("background")[0], format="xml") except IndexError: - self.background = "" + self.background = "" # finally the interesting tags (product, affected, package) self.glsatype = myroot.getElementsByTagName("product")[0].getAttribute("type") @@ -572,16 +594,18 @@ class Glsa: self.services = self.affected.getElementsByTagName("service") return None - def dump(self, outstream=sys.stdout): + def dump(self, outstream=sys.stdout, encoding="utf-8"): """ - Dumps a plaintext representation of this GLSA to I{outfile} or + Dumps a plaintext representation of this GLSA to I{outfile} or B{stdout} if it is ommitted. You can specify an alternate - I{encoding} if needed (default is latin1). - + I{encoding} if needed (default is utf-8). + @type outstream: File @param outfile: Stream that should be used for writing (defaults to sys.stdout) """ + outstream = getattr(outstream, "buffer", outstream) + outstream = codecs.getwriter(encoding)(outstream) width = 76 outstream.write(("GLSA %s: \n%s" % (self.nr, self.title)).center(width)+"\n") outstream.write((width*"=")+"\n") @@ -606,30 +630,24 @@ class Glsa: pass if len(self.bugs) > 0: outstream.write(_("\nRelated bugs: ")) - for i in range(0, len(self.bugs)): - outstream.write(self.bugs[i]) - if i < len(self.bugs)-1: - outstream.write(", ") - else: - outstream.write("\n") + outstream.write(", ".join(self.bugs)) + outstream.write("\n") if self.background: outstream.write("\n"+wrap(self.background, width, caption=_("Background: "))) outstream.write("\n"+wrap(self.description, width, caption=_("Description: "))) outstream.write("\n"+wrap(self.impact_text, width, caption=_("Impact: "))) outstream.write("\n"+wrap(self.workaround, width, caption=_("Workaround: "))) outstream.write("\n"+wrap(self.resolution, width, caption=_("Resolution: "))) - myreferences = "" - for r in self.references: - myreferences += (r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE+" ") + myreferences = " ".join(r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE for r in self.references) outstream.write("\n"+wrap(myreferences, width, caption=_("References: "))) outstream.write("\n") - + def isVulnerable(self): """ Tests if the system is affected by this GLSA by checking if any vulnerable package versions are installed. Also checks for affected architectures. - + @rtype: Boolean @return: True if the system is affected, False if not """ @@ -641,56 +659,67 @@ class Glsa: for v in path["vul_atoms"]: rValue = rValue \ or (len(match(v, self.vardbapi)) > 0 \ - and getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + and None != getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ self.portdbapi, self.vardbapi)) return rValue - - def isApplied(self): + + def isInjected(self): """ - Looks if the GLSA IDis in the GLSA checkfile to check if this - GLSA was already applied. - + Looks if the GLSA ID is in the GLSA checkfile to check if this + GLSA should be marked as applied. + @rtype: Boolean - @return: True if the GLSA was applied, False if not + @returns: True if the GLSA is in the inject file, False if not """ + if not os.access(os.path.join(self.config["EROOT"], + PRIVATE_PATH, "glsa_injected"), os.R_OK): + return False return (self.nr in get_applied_glsas(self.config)) def inject(self): """ Puts the ID of this GLSA into the GLSA checkfile, so it won't - show up on future checks. Should be called after a GLSA is + show up on future checks. Should be called after a GLSA is applied or on explicit user request. @rtype: None @return: None """ - if not self.isApplied(): + if not self.isInjected(): checkfile = io.open( _unicode_encode(os.path.join(self.config["EROOT"], - CACHE_PATH, "glsa"), - encoding=_encodings['fs'], errors='strict'), + PRIVATE_PATH, "glsa_injected"), + encoding=_encodings['fs'], errors='strict'), mode='a+', encoding=_encodings['content'], errors='strict') checkfile.write(_unicode_decode(self.nr + "\n")) checkfile.close() return None - + def getMergeList(self, least_change=True): """ Returns the list of package-versions that have to be merged to - apply this GLSA properly. The versions are as low as possible + apply this GLSA properly. The versions are as low as possible while avoiding downgrades (see L{getMinUpgrade}). - + @type least_change: Boolean @param least_change: True if the smallest possible upgrade should be selected, False for an emerge-like algorithm @rtype: List of Strings @return: list of package-versions that have to be merged """ - rValue = [] - for pkg in self.packages: + return list(set(update for (vuln, update) in self.getAffectionTable(least_change) if update)) + + def getAffectionTable(self, least_change=True): + """ + Will initialize the self.systemAffection list of + atoms installed on the system that are affected + by this GLSA, and the atoms that are minimal upgrades. + """ + systemAffection = [] + for pkg in self.packages.keys(): for path in self.packages[pkg]: - update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], self.portdbapi, self.vardbapi, minimize=least_change) if update: - rValue.append(update) - return rValue + systemAffection.extend(update) + return systemAffection diff --git a/portage_with_autodep/pym/portage/glsa.pyo b/portage_with_autodep/pym/portage/glsa.pyo Binary files differindex 65162f1..dd97e49 100644 --- a/portage_with_autodep/pym/portage/glsa.pyo +++ b/portage_with_autodep/pym/portage/glsa.pyo diff --git a/portage_with_autodep/pym/portage/localization.py b/portage_with_autodep/pym/portage/localization.py index d16c4b1..2815ef5 100644 --- a/portage_with_autodep/pym/portage/localization.py +++ b/portage_with_autodep/pym/portage/localization.py @@ -1,12 +1,18 @@ # localization.py -- Code to manage/help portage localization. -# Copyright 2004 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from portage import _unicode_decode # We define this to make the transition easier for us. def _(mystr): - return mystr - + """ + Always returns unicode, regardless of the input type. This is + helpful for avoiding UnicodeDecodeError from __str__() with + Python 2, by ensuring that string format operations invoke + __unicode__() instead of __str__(). + """ + return _unicode_decode(mystr) def localization_example(): # Dict references allow translators to rearrange word order. diff --git a/portage_with_autodep/pym/portage/localization.pyo b/portage_with_autodep/pym/portage/localization.pyo Binary files differindex e992e3a..50a2805 100644 --- a/portage_with_autodep/pym/portage/localization.pyo +++ b/portage_with_autodep/pym/portage/localization.pyo diff --git a/portage_with_autodep/pym/portage/locks.py b/portage_with_autodep/pym/portage/locks.py index 59fbc6e..8571d8c 100644 --- a/portage_with_autodep/pym/portage/locks.py +++ b/portage_with_autodep/pym/portage/locks.py @@ -1,5 +1,5 @@ # portage: Lock management code -# Copyright 2004-2012 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ @@ -17,7 +17,6 @@ import portage from portage import os, _encodings, _unicode_decode from portage.exception import DirectoryNotFound, FileNotFound, \ InvalidData, TryAgain, OperationNotPermitted, PermissionDenied -from portage.data import portage_gid from portage.util import writemsg from portage.localization import _ @@ -64,6 +63,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if not mypath: raise InvalidData(_("Empty path given")) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors @@ -148,7 +150,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, except IOError as e: if not hasattr(e, "errno"): raise - if e.errno in (errno.EACCES, errno.EAGAIN): + if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) @@ -163,19 +165,43 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: - waiting_msg = _("waiting for lock on %s\n") % lockfilename + waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. - try: - locking_method(myfd, fcntl.LOCK_EX) - except EnvironmentError as e: - if out is not None: - out.eend(1, str(e)) - raise + enolock_msg_shown = False + while True: + try: + locking_method(myfd, fcntl.LOCK_EX) + except EnvironmentError as e: + if e.errno == errno.ENOLCK: + # This is known to occur on Solaris NFS (see + # bug #462694). Assume that the error is due + # to temporary exhaustion of record locks, + # and loop until one becomes available. + if not enolock_msg_shown: + enolock_msg_shown = True + if isinstance(mypath, int): + context_desc = _("Error while waiting " + "to lock fd %i") % myfd + else: + context_desc = _("Error while waiting " + "to lock '%s'") % lockfilename + writemsg("\n!!! %s: %s\n" % (context_desc, e), + noiselevel=-1) + + time.sleep(_HARDLINK_POLL_LATENCY) + continue + + if out is not None: + out.eend(1, str(e)) + raise + else: + break + if out is not None: out.eend(os.EX_OK) - elif e.errno in (errno.ENOSYS, errno.ENOLCK): + elif e.errno in (errno.ENOSYS,): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe @@ -207,6 +233,17 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(myfd, fcntl.F_SETFD, + fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + _open_fds.add(myfd) writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1) @@ -339,6 +376,9 @@ def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: diff --git a/portage_with_autodep/pym/portage/locks.pyo b/portage_with_autodep/pym/portage/locks.pyo Binary files differindex 9c90a2f..c2e24fb 100644 --- a/portage_with_autodep/pym/portage/locks.pyo +++ b/portage_with_autodep/pym/portage/locks.pyo diff --git a/portage_with_autodep/pym/portage/mail.pyo b/portage_with_autodep/pym/portage/mail.pyo Binary files differindex bc3a76d..f0ef5df 100644 --- a/portage_with_autodep/pym/portage/mail.pyo +++ b/portage_with_autodep/pym/portage/mail.pyo diff --git a/portage_with_autodep/pym/portage/manifest.py b/portage_with_autodep/pym/portage/manifest.py index 90324ee..510e203 100644 --- a/portage_with_autodep/pym/portage/manifest.py +++ b/portage_with_autodep/pym/portage/manifest.py @@ -1,14 +1,19 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re +import sys import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes', + 'portage.repository.config:_find_invalid_path_char', 'portage.util:write_atomic', ) @@ -23,8 +28,15 @@ from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS, MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH) from portage.localization import _ -# Characters prohibited by repoman's file.name check. -_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') +_manifest_re = re.compile( + r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (.*)( \d+( \S+ \S+)+)$', + re.UNICODE) + +if sys.hexversion >= 0x3000000: + _unicode = str + basestring = str +else: + _unicode = unicode class FileNotInManifestException(PortageException): pass @@ -37,15 +49,10 @@ def manifest2AuxfileFilter(filename): for x in mysplit: if x[:1] == '.': return False - if _prohibited_filename_chars_re.search(x) is not None: - return False return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): - filename = filename.strip(os.sep) - if _prohibited_filename_chars_re.search(filename) is not None: - return False - return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) + return not (filename == "Manifest" or filename.endswith(".ebuild")) def guessManifestFileType(filename): """ Perform a best effort guess of which type the given filename is, avoid using this if possible """ @@ -66,18 +73,17 @@ def guessThinManifestFileType(filename): return None return "DIST" -def parseManifest2(mysplit): +def parseManifest2(line): + if not isinstance(line, basestring): + line = ' '.join(line) myentry = None - if len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS: - mytype = mysplit[0] - myname = mysplit[1] - try: - mysize = int(mysplit[2]) - except ValueError: - return None - myhashes = dict(zip(mysplit[3::2], mysplit[4::2])) - myhashes["size"] = mysize - myentry = Manifest2Entry(type=mytype, name=myname, hashes=myhashes) + match = _manifest_re.match(line) + if match is not None: + tokens = match.group(3).split() + hashes = dict(zip(tokens[1::2], tokens[2::2])) + hashes["size"] = int(tokens[0]) + myentry = Manifest2Entry(type=match.group(1), + name=match.group(2), hashes=hashes) return myentry class ManifestEntry(object): @@ -107,11 +113,20 @@ class Manifest2Entry(ManifestEntry): def __ne__(self, other): return not self.__eq__(other) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['repo.content'], errors='strict') + class Manifest(object): parsers = (parseManifest2,) - def __init__(self, pkgdir, distdir, fetchlist_dict=None, + def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, - allow_missing=False, allow_create=True, hashes=None): + allow_missing=False, allow_create=True, hashes=None, + find_invalid_path_char=None): """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of @@ -124,6 +139,9 @@ class Manifest(object): "portage.manifest.Manifest constructor is deprecated.", DeprecationWarning, stacklevel=2) + if find_invalid_path_char is None: + find_invalid_path_char = _find_invalid_path_char + self._find_invalid_path_char = find_invalid_path_char self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() @@ -172,13 +190,12 @@ class Manifest(object): """Parse a manifest. If myhashdict is given then data will be added too it. Otherwise, a new dict will be created and returned.""" try: - fd = io.open(_unicode_encode(file_path, + with io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['repo.content'], errors='replace') - if myhashdict is None: - myhashdict = {} - self._parseDigests(fd, myhashdict=myhashdict, **kwargs) - fd.close() + encoding=_encodings['repo.content'], errors='replace') as f: + if myhashdict is None: + myhashdict = {} + self._parseDigests(f, myhashdict=myhashdict, **kwargs) return myhashdict except (OSError, IOError) as e: if e.errno == errno.ENOENT: @@ -197,9 +214,8 @@ class Manifest(object): """Parse manifest lines and return a list of manifest entries.""" for myline in mylines: myentry = None - mysplit = myline.split() for parser in self.parsers: - myentry = parser(mysplit) + myentry = parser(myline) if myentry is not None: yield myentry break # go to the next line @@ -254,9 +270,12 @@ class Manifest(object): (MANIFEST2_REQUIRED_HASH, t, f)) def write(self, sign=False, force=False): - """ Write Manifest instance to disk, optionally signing it """ + """ Write Manifest instance to disk, optionally signing it. Returns + True if the Manifest is actually written, and False if the write + is skipped due to existing Manifest being identical.""" + rval = False if not self.allow_create: - return + return rval self.checkIntegrity() try: myentries = list(self._createManifestEntries()) @@ -288,7 +307,8 @@ class Manifest(object): # thin manifests with no DIST entries, myentries is # non-empty for all currently known use cases. write_atomic(self.getFullname(), "".join("%s\n" % - str(myentry) for myentry in myentries)) + _unicode(myentry) for myentry in myentries)) + rval = True else: # With thin manifest, there's no need to have # a Manifest file if there are no DIST entries. @@ -297,6 +317,7 @@ class Manifest(object): except OSError as e: if e.errno != errno.ENOENT: raise + rval = True if sign: self.sign() @@ -304,6 +325,7 @@ class Manifest(object): if e.errno == errno.EACCES: raise PermissionDenied(str(e)) raise + return rval def sign(self): """ Sign the Manifest """ @@ -362,10 +384,11 @@ class Manifest(object): distfilehashes = self.fhashdict["DIST"] else: distfilehashes = {} - self.__init__(self.pkgdir, self.distdir, + self.__init__(self.pkgdir, distdir=self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, thin=self.thin, allow_missing=self.allow_missing, - allow_create=self.allow_create, hashes=self.hashes) + allow_create=self.allow_create, hashes=self.hashes, + find_invalid_path_char=self._find_invalid_path_char) pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() @@ -460,7 +483,8 @@ class Manifest(object): if pf is not None: mytype = "EBUILD" cpvlist.append(pf) - elif manifest2MiscfileFilter(f): + elif self._find_invalid_path_char(f) == -1 and \ + manifest2MiscfileFilter(f): mytype = "MISC" else: continue @@ -479,7 +503,8 @@ class Manifest(object): full_path = os.path.join(parentdir, f) recursive_files.append(full_path[cut_len:]) for f in recursive_files: - if not manifest2AuxfileFilter(f): + if self._find_invalid_path_char(f) != -1 or \ + not manifest2AuxfileFilter(f): continue self.fhashdict["AUX"][f] = perform_multiple_checksums( os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes) @@ -501,14 +526,17 @@ class Manifest(object): for t in MANIFEST2_IDENTIFIERS: self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles) - def checkTypeHashes(self, idtype, ignoreMissingFiles=False): + def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None): for f in self.fhashdict[idtype]: - self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles) + self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles, + hash_filter=hash_filter) - def checkFileHashes(self, ftype, fname, ignoreMissing=False): - myhashes = self.fhashdict[ftype][fname] + def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None): + digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) try: - ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + ok, reason = verify_all(self._getAbsname(ftype, fname), digests) if not ok: raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) return ok, reason diff --git a/portage_with_autodep/pym/portage/manifest.pyo b/portage_with_autodep/pym/portage/manifest.pyo Binary files differindex d482bbd..40d96e1 100644 --- a/portage_with_autodep/pym/portage/manifest.pyo +++ b/portage_with_autodep/pym/portage/manifest.pyo diff --git a/portage_with_autodep/pym/portage/news.py b/portage_with_autodep/pym/portage/news.py index bbd9325..408fb5c 100644 --- a/portage_with_autodep/pym/portage/news.py +++ b/portage_with_autodep/pym/portage/news.py @@ -1,8 +1,8 @@ # portage: news management code -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", @@ -13,6 +13,7 @@ import io import logging import os as _os import re +import portage from portage import OrderedDict from portage import os from portage import _encodings @@ -241,7 +242,8 @@ class NewsItem(object): for values in self.restrictions.values(): any_match = False for restriction in values: - if restriction.checkRestriction(**kwargs): + if restriction.checkRestriction( + **portage._native_kwargs(kwargs)): any_match = True if not any_match: all_match = False @@ -388,7 +390,7 @@ def count_unread_news(portdb, vardb, repos=None, update=True): # NOTE: The NewsManager typically handles permission errors by # returning silently, so PermissionDenied won't necessarily be # raised even if we do trigger a permission error above. - msg = _unicode_decode("Permission denied: '%s'\n") % (e,) + msg = "Permission denied: '%s'\n" % (e,) if msg in permission_msgs: pass else: diff --git a/portage_with_autodep/pym/portage/news.pyo b/portage_with_autodep/pym/portage/news.pyo Binary files differindex bbd247c..0a1d4ab 100644 --- a/portage_with_autodep/pym/portage/news.pyo +++ b/portage_with_autodep/pym/portage/news.pyo diff --git a/portage_with_autodep/pym/portage/output.py b/portage_with_autodep/pym/portage/output.py index 98bec81..fc1b042 100644 --- a/portage_with_autodep/pym/portage/output.py +++ b/portage_with_autodep/pym/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __docformat__ = "epytext" @@ -7,6 +7,7 @@ import errno import io import formatter import re +import subprocess import sys import portage @@ -163,15 +164,12 @@ def _parse_color_map(config_root='/', onerror=None): token = token[1:-1] return token - f = None try: - f = io.open(_unicode_encode(myfile, + with io.open(_unicode_encode(myfile, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - lineno = 0 - for line in f: - lineno += 1 - + mode='r', encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() + for lineno, line in enumerate(lines): commenter_pos = line.find("#") line = line[:commenter_pos].strip() @@ -229,9 +227,6 @@ def _parse_color_map(config_root='/', onerror=None): elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise - finally: - if f is not None: - f.close() def nc_len(mystr): tmp = re.sub(esc_seq + "^m]+m", "", mystr); @@ -244,7 +239,7 @@ _max_xtermTitle_len = 253 def xtermTitle(mystr, raw=False): global _disable_xtermTitle if _disable_xtermTitle is None: - _disable_xtermTitle = not (sys.stderr.isatty() and \ + _disable_xtermTitle = not (sys.__stderr__.isatty() and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None) @@ -277,15 +272,18 @@ def xtermTitleReset(): if dotitles and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None and \ - sys.stderr.isatty(): + sys.__stderr__.isatty(): from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") if shell: spawn([shell, "-c", prompt_command], env=os.environ, - fd_pipes={0:sys.stdin.fileno(),1:sys.stderr.fileno(), - 2:sys.stderr.fileno()}) + fd_pipes={ + 0: portage._get_stdin().fileno(), + 1: sys.__stderr__.fileno(), + 2: sys.__stderr__.fileno() + }) else: os.system(prompt_command) return @@ -425,28 +423,41 @@ class StyleWriter(formatter.DumbWriter): if self.style_listener: self.style_listener(styles) -def get_term_size(): +def get_term_size(fd=None): """ Get the number of lines and columns of the tty that is connected to - stdout. Returns a tuple of (lines, columns) or (0, 0) if an error + fd. Returns a tuple of (lines, columns) or (0, 0) if an error occurs. The curses module is used if available, otherwise the output of `stty size` is parsed. The lines and columns values are guaranteed to be greater than or equal to zero, since a negative COLUMNS variable is known to prevent some commands from working (see bug #394091). """ - if not sys.stdout.isatty(): + if fd is None: + fd = sys.stdout + if not hasattr(fd, 'isatty') or not fd.isatty(): return (0, 0) try: import curses try: - curses.setupterm() + curses.setupterm(term=os.environ.get("TERM", "unknown"), + fd=fd.fileno()) return curses.tigetnum('lines'), curses.tigetnum('cols') except curses.error: pass except ImportError: pass - st, out = portage.subprocess_getstatusoutput('stty size') - if st == os.EX_OK: + + try: + proc = subprocess.Popen(["stty", "size"], + stdout=subprocess.PIPE, stderr=fd) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + # stty command not found + return (0, 0) + + out = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: out = out.split() if len(out) == 2: try: @@ -631,11 +642,14 @@ class EOutput(object): class ProgressBar(object): """The interface is copied from the ProgressBar class from the EasyDialogs module (which is Mac only).""" - def __init__(self, title=None, maxval=0, label=None): - self._title = title + def __init__(self, title=None, maxval=0, label=None, max_desc_length=25): + self._title = title or "" self._maxval = maxval - self._label = maxval + self._label = label or "" self._curval = 0 + self._desc = "" + self._desc_max_length = max_desc_length + self._set_desc() @property def curval(self): @@ -659,10 +673,23 @@ class ProgressBar(object): def title(self, newstr): """Sets the text in the title bar of the progress dialog to newstr.""" self._title = newstr + self._set_desc() def label(self, newstr): """Sets the text in the progress box of the progress dialog to newstr.""" self._label = newstr + self._set_desc() + + def _set_desc(self): + self._desc = "%s%s" % ( + "%s: " % self._title if self._title else "", + "%s" % self._label if self._label else "" + ) + if len(self._desc) > self._desc_max_length: # truncate if too long + self._desc = "%s..." % self._desc[:self._desc_max_length - 3] + if len(self._desc): + self._desc = self._desc.ljust(self._desc_max_length) + def set(self, value, maxval=None): """ @@ -691,10 +718,10 @@ class ProgressBar(object): class TermProgressBar(ProgressBar): """A tty progress bar similar to wget's.""" - def __init__(self, **kwargs): + def __init__(self, fd=sys.stdout, **kwargs): ProgressBar.__init__(self, **kwargs) - lines, self.term_columns = get_term_size() - self.file = sys.stdout + lines, self.term_columns = get_term_size(fd) + self.file = fd self._min_columns = 11 self._max_columns = 80 # for indeterminate mode, ranges from 0.0 to 1.0 @@ -717,16 +744,18 @@ class TermProgressBar(ProgressBar): curval = self._curval maxval = self._maxval position = self._position - percentage_str_width = 4 + percentage_str_width = 5 square_brackets_width = 2 if cols < percentage_str_width: return "" - bar_space = cols - percentage_str_width - square_brackets_width + bar_space = cols - percentage_str_width - square_brackets_width - 1 + if self._desc: + bar_space -= self._desc_max_length if maxval == 0: max_bar_width = bar_space-3 - image = " " + _percent = "".ljust(percentage_str_width) if cols < min_columns: - return image + return "" if position <= 0.5: offset = 2 * position else: @@ -742,16 +771,16 @@ class TermProgressBar(ProgressBar): position = 0.5 self._position = position bar_width = int(offset * max_bar_width) - image = image + "[" + (bar_width * " ") + \ - "<=>" + ((max_bar_width - bar_width) * " ") + "]" + image = "%s%s%s" % (self._desc, _percent, + "[" + (bar_width * " ") + \ + "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: percentage = int(100 * float(curval) / maxval) - if percentage == 100: - percentage_str_width += 1 - bar_space -= 1 max_bar_width = bar_space - 1 - image = ("%d%% " % percentage).rjust(percentage_str_width) + _percent = ("%d%% " % percentage).rjust(percentage_str_width) + image = "%s%s" % (self._desc, _percent) + if cols < min_columns: return image offset = float(curval) / maxval diff --git a/portage_with_autodep/pym/portage/output.pyo b/portage_with_autodep/pym/portage/output.pyo Binary files differindex 993a2de..2bb98b0 100644 --- a/portage_with_autodep/pym/portage/output.pyo +++ b/portage_with_autodep/pym/portage/output.pyo diff --git a/portage_with_autodep/pym/portage/package/__init__.pyo b/portage_with_autodep/pym/portage/package/__init__.pyo Binary files differindex 9d8f30c..a17fc87 100644 --- a/portage_with_autodep/pym/portage/package/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo Binary files differindex 927b4bc..ff34626 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py index 0c613ce..af606f1 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.versions import _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -77,7 +77,9 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: pkg = _pkg_str(cpv, slot=slot, repo=repo) else: pkg = cpv @@ -91,6 +93,47 @@ class KeywordsManager(object): keywords.extend(pkg_keywords) return stack_lists(keywords, incremental=True) + def isStable(self, pkg, global_accept_keywords, backuped_accept_keywords): + mygroups = self.getKeywords(pkg, None, pkg._metadata["KEYWORDS"], None) + pgroups = global_accept_keywords.split() + + unmaskgroups = self.getPKeywords(pkg, None, None, + global_accept_keywords) + pgroups.extend(unmaskgroups) + + egroups = backuped_accept_keywords.split() + + if unmaskgroups or egroups: + pgroups = self._getEgroups(egroups, pgroups) + else: + pgroups = set(pgroups) + + if self._getMissingKeywords(pkg, pgroups, mygroups): + return False + + if pkg.cpv._settings.local_config: + # If replacing all keywords with unstable variants would mask the + # package, then it's considered stable. + unstable = [] + for kw in mygroups: + if kw[:1] != "~": + kw = "~" + kw + unstable.append(kw) + + return bool(self._getMissingKeywords(pkg, pgroups, set(unstable))) + else: + # For repoman, if the package has an effective stable keyword that + # intersects with the effective ACCEPT_KEYWORDS for the current + # profile, then consider it stable. + for kw in pgroups: + if kw[:1] != "~": + if kw in mygroups or '*' in mygroups: + return True + if kw == '*': + for x in mygroups: + if x[:1] != "~": + return True + return False def getMissingKeywords(self, cpv, @@ -237,7 +280,7 @@ class KeywordsManager(object): if not mygroups: # If KEYWORDS is empty then we still have to return something # in order to distinguish from the case of "none missing". - mygroups.append("**") + mygroups = ["**"] missing = mygroups return missing @@ -261,9 +304,11 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: cpv = _pkg_str(cpv, slot=slot, repo=repo) - cp = cpv_getkey(cpv) + cp = cpv.cp unmaskgroups = [] if self._p_accept_keywords: @@ -288,4 +333,3 @@ class KeywordsManager(object): for x in pkg_accept_keywords: unmaskgroups.extend(x) return unmaskgroups - diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo Binary files differindex 15043f0..b922211 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo Binary files differindex 4a38298..4ddda5e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py index f7a1177..80b6a70 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'LocationsManager', ) @@ -13,10 +15,12 @@ import portage from portage import os, eapi_is_supported, _encodings, _unicode_encode from portage.const import CUSTOM_PROFILE_PATH, GLOBAL_CONFIG_PATH, \ PROFILE_PATH, USER_CONFIG_PATH +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.repository.config import parse_layout_conf, \ _portage1_profiles_allow_directories @@ -27,7 +31,7 @@ _PORTAGE1_DIRECTORIES = frozenset([ 'use.mask', 'use.force']) _profile_node = collections.namedtuple('_profile_node', - 'location portage1_directories') + 'location portage1_directories user_config') _allow_parent_colon = frozenset( ["portage-2"]) @@ -45,9 +49,13 @@ class LocationsManager(object): if self.eprefix is None: self.eprefix = portage.const.EPREFIX + elif self.eprefix: + self.eprefix = normalize_path(self.eprefix) + if self.eprefix == os.sep: + self.eprefix = "" if self.config_root is None: - self.config_root = self.eprefix + os.sep + self.config_root = portage.const.EPREFIX + os.sep self.config_root = normalize_path(os.path.abspath( self.config_root)).rstrip(os.path.sep) + os.path.sep @@ -72,14 +80,26 @@ class LocationsManager(object): known_repos = tuple(known_repos) if self.config_profile_path is None: + deprecated_profile_path = os.path.join( + self.config_root, 'etc', 'make.profile') self.config_profile_path = \ os.path.join(self.config_root, PROFILE_PATH) - if os.path.isdir(self.config_profile_path): + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path + if isdir_raise_eaccess(deprecated_profile_path) and not \ + os.path.samefile(self.profile_path, + deprecated_profile_path): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found 2 make.profile dirs: " + "using '%s', ignoring '%s'") % + (self.profile_path, deprecated_profile_path), + noiselevel=-1) else: - self.config_profile_path = \ - os.path.join(self.abs_user_config, 'make.profile') - if os.path.isdir(self.config_profile_path): + self.config_profile_path = deprecated_profile_path + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path else: self.profile_path = None @@ -99,9 +119,9 @@ class LocationsManager(object): self._addProfile(os.path.realpath(self.profile_path), repositories, known_repos) except ParseError as e: - writemsg(_("!!! Unable to parse profile: '%s'\n") % \ - self.profile_path, noiselevel=-1) - writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) + if not portage._sync_disabled_warnings: + writemsg(_("!!! Unable to parse profile: '%s'\n") % self.profile_path, noiselevel=-1) + writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] self.profiles_complex = [] @@ -111,14 +131,15 @@ class LocationsManager(object): if os.path.exists(custom_prof): self.user_profile_dir = custom_prof self.profiles.append(custom_prof) - self.profiles_complex.append(_profile_node(custom_prof, True)) + self.profiles_complex.append( + _profile_node(custom_prof, True, True)) del custom_prof self.profiles = tuple(self.profiles) self.profiles_complex = tuple(self.profiles_complex) def _check_var_directory(self, varname, var): - if not os.path.isdir(var): + if not isdir_raise_eaccess(var): writemsg(_("!!! Error: %s='%s' is not a directory. " "Please correct this.\n") % (varname, var), noiselevel=-1) @@ -130,33 +151,9 @@ class LocationsManager(object): allow_parent_colon = True repo_loc = None compat_mode = False - intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] - if intersecting_repos: - # protect against nested repositories. Insane configuration, but the longest - # path will be the correct one. - repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) - allow_directories = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) - allow_parent_colon = any(x in _allow_parent_colon - for x in layout_data['profile-formats']) - if compat_mode: - offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) - offenders = sorted(x for x in offenders - if os.path.isdir(os.path.join(currentPath, x))) - if offenders: - warnings.warn(_("Profile '%(profile_path)s' in repository " - "'%(repo_name)s' is implicitly using 'portage-1' profile format, but " - "the repository profiles are not marked as that format. This will break " - "in the future. Please either convert the following paths " - "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf. Files: '%(files)s'\n") - % dict(profile_path=currentPath, repo_name=repo_loc, - files=', '.join(offenders))) - - parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") + eapi = "0" f = None try: f = io.open(_unicode_encode(eapi_file, @@ -174,7 +171,38 @@ class LocationsManager(object): finally: if f is not None: f.close() - if os.path.exists(parentsFile): + + intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] + if intersecting_repos: + # protect against nested repositories. Insane configuration, but the longest + # path will be the correct one. + repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) + allow_directories = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + compat_mode = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) + + if compat_mode: + offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) + offenders = sorted(x for x in offenders + if os.path.isdir(os.path.join(currentPath, x))) + if offenders: + warnings.warn(_( + "\nThe selected profile is implicitly using the 'portage-1' format:\n" + "\tprofile = %(profile_path)s\n" + "But this repository is not using that format:\n" + "\trepo = %(repo_name)s\n" + "This will break in the future. Please convert these dirs to files:\n" + "\t%(files)s\n" + "Or, add this line to the repository's layout.conf:\n" + "\tprofile-formats = portage-1") + % dict(profile_path=currentPath, repo_name=repo_loc, + files='\n\t'.join(offenders))) + + parentsFile = os.path.join(currentPath, "parent") + if exists_raise_eaccess(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( @@ -196,7 +224,7 @@ class LocationsManager(object): # of the current repo, so realpath it. parentPath = os.path.realpath(parentPath) - if os.path.exists(parentPath): + if exists_raise_eaccess(parentPath): self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( @@ -205,7 +233,7 @@ class LocationsManager(object): self.profiles.append(currentPath) self.profiles_complex.append( - _profile_node(currentPath, allow_directories)) + _profile_node(currentPath, allow_directories, False)) def _expand_parent_colon(self, parentsFile, parentPath, repo_loc, repositories): @@ -253,29 +281,10 @@ class LocationsManager(object): self.eroot = self.target_root.rstrip(os.sep) + self.eprefix + os.sep - # make.globals should not be relative to config_root - # because it only contains constants. However, if EPREFIX - # is set then there are two possible scenarios: - # 1) If $ROOT == "/" then make.globals should be - # relative to EPREFIX. - # 2) If $ROOT != "/" then the correct location of - # make.globals needs to be specified in the constructor - # parameters, since it's a property of the host system - # (and the current config represents the target system). self.global_config_path = GLOBAL_CONFIG_PATH - if self.eprefix: - if self.target_root == "/": - # case (1) above - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume make.globals is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.global_config_path = os.path.join(portage.const.EPREFIX, + GLOBAL_CONFIG_PATH.lstrip(os.sep)) def set_port_dirs(self, portdir, portdir_overlay): self.portdir = portdir @@ -287,7 +296,7 @@ class LocationsManager(object): for ov in shlex_split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") - if os.path.isdir(profiles_dir): + if isdir_raise_eaccess(profiles_dir): self.overlay_profiles.append(profiles_dir) self.profile_locations = [os.path.join(portdir, "profiles")] + self.overlay_profiles diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo Binary files differindex c64d313..5ef52ec 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py index bce1152..aeb04d7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -8,11 +8,10 @@ __all__ = ( import warnings from portage import os -from portage.dep import ExtendedAtomDict, match_from_list, _repo_separator, _slot_separator +from portage.dep import ExtendedAtomDict, match_from_list from portage.localization import _ from portage.util import append_repo, grabfile_package, stack_lists, writemsg -from portage.versions import cpv_getkey -from _emerge.Package import Package +from portage.versions import _pkg_str class MaskManager(object): @@ -47,7 +46,7 @@ class MaskManager(object): "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf.\n") + "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc] @@ -185,12 +184,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - mask_atoms = self._pmaskdict.get(cp) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + mask_atoms = self._pmaskdict.get(pkg.cp) if mask_atoms: - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) pkg_list = [pkg] for x in mask_atoms: if not match_from_list(x, pkg_list): @@ -219,8 +221,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - return self._getMaskAtom(cpv, slot, repo, self._punmaskdict.get(cp)) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + return self._getMaskAtom(pkg, slot, repo, + self._punmaskdict.get(pkg.cp)) def getRawMaskAtom(self, cpv, slot, repo): diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo Binary files differindex f48eb47..abc46de 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py index e1ec7f4..0d00810 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,36 +7,49 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import Atom, dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re, _repo_separator +from portage.eapi import eapi_has_use_aliases, eapi_supports_stable_use_forcing_and_masking +from portage.exception import InvalidAtom from portage.localization import _ -from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.util import grabfile, grabdict, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg +from portage.versions import _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity class UseManager(object): - def __init__(self, repositories, profiles, abs_user_config, user_config=True): + def __init__(self, repositories, profiles, abs_user_config, is_stable, + user_config=True): # file variable #-------------------------------- # repositories #-------------------------------- # use.mask _repo_usemask_dict + # use.stable.mask _repo_usestablemask_dict # use.force _repo_useforce_dict + # use.stable.force _repo_usestableforce_dict + # use.aliases _repo_usealiases_dict # package.use.mask _repo_pusemask_dict + # package.use.stable.mask _repo_pusestablemask_dict # package.use.force _repo_puseforce_dict + # package.use.stable.force _repo_pusestableforce_dict + # package.use.aliases _repo_pusealiases_dict #-------------------------------- # profiles #-------------------------------- # use.mask _usemask_list + # use.stable.mask _usestablemask_list # use.force _useforce_list + # use.stable.force _usestableforce_list # package.use.mask _pusemask_list + # package.use.stable.mask _pusestablemask_list # package.use _pkgprofileuse # package.use.force _puseforce_list + # package.use.stable.force _pusestableforce_list #-------------------------------- # user config #-------------------------------- - # package.use _pusedict + # package.use _pusedict # Dynamic variables tracked by the config class #-------------------------------- @@ -49,26 +62,61 @@ class UseManager(object): #-------------------------------- # puse + self._user_config = user_config + self._is_stable = is_stable self._repo_usemask_dict = self._parse_repository_files_to_dict_of_tuples("use.mask", repositories) + self._repo_usestablemask_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_useforce_dict = self._parse_repository_files_to_dict_of_tuples("use.force", repositories) + self._repo_usestableforce_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_pusemask_dict = self._parse_repository_files_to_dict_of_dicts("package.use.mask", repositories) + self._repo_pusestablemask_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puseforce_dict = self._parse_repository_files_to_dict_of_dicts("package.use.force", repositories) + self._repo_pusestableforce_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puse_dict = self._parse_repository_files_to_dict_of_dicts("package.use", repositories) self._usemask_list = self._parse_profile_files_to_tuple_of_tuples("use.mask", profiles) + self._usestablemask_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._useforce_list = self._parse_profile_files_to_tuple_of_tuples("use.force", profiles) + self._usestableforce_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusemask_list = self._parse_profile_files_to_tuple_of_dicts("package.use.mask", profiles) + self._pusestablemask_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pkgprofileuse = self._parse_profile_files_to_tuple_of_dicts("package.use", profiles, juststrings=True) self._puseforce_list = self._parse_profile_files_to_tuple_of_dicts("package.use.force", profiles) + self._pusestableforce_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusedict = self._parse_user_files_to_extatomdict("package.use", abs_user_config, user_config) + self._repo_usealiases_dict = self._parse_repository_usealiases(repositories) + self._repo_pusealiases_dict = self._parse_repository_packageusealiases(repositories) + self.repositories = repositories - - def _parse_file_to_tuple(self, file_name, recursive=True): + + def _parse_file_to_tuple(self, file_name, recursive=True, eapi_filter=None): ret = [] lines = grabfile(file_name, recursive=recursive) eapi = read_corresponding_eapi_file(file_name) + if eapi_filter is not None and not eapi_filter(eapi): + if lines: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return () useflag_re = _get_useflag_re(eapi) for prefixed_useflag in lines: if prefixed_useflag[:1] == "-": @@ -82,11 +130,26 @@ class UseManager(object): ret.append(prefixed_useflag) return tuple(ret) - def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True): + def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True, + eapi_filter=None, user_config=False): ret = {} location_dict = {} - file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) - eapi = read_corresponding_eapi_file(file_name) + eapi = read_corresponding_eapi_file(file_name, default=None) + if eapi is None and not user_config: + eapi = "0" + if eapi is None: + ret = ExtendedAtomDict(dict) + else: + ret = {} + file_dict = grabdict_package(file_name, recursive=recursive, + allow_wildcard=(eapi is None), allow_repo=(eapi is None), + verify_eapi=(eapi is not None)) + if eapi is not None and eapi_filter is not None and not eapi_filter(eapi): + if file_dict: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return ret useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): useflags = [] @@ -119,31 +182,116 @@ class UseManager(object): return ret - def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories): + def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories): + def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations): + def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations, + eapi_filter=None): return tuple(self._parse_file_to_tuple( os.path.join(profile.location, file_name), - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter) for profile in locations) - def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False): + def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, + juststrings=False, eapi_filter=None): return tuple(self._parse_file_to_dict( os.path.join(profile.location, file_name), juststrings, - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter, + user_config=profile.user_config) for profile in locations) - def getUseMask(self, pkg=None): + def _parse_repository_usealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + raw_file_dict = grabdict(file_name, recursive=True) + file_dict = {} + for real_flag, aliases in raw_file_dict.items(): + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag in '%s': '%s'\n") % (file_name, real_flag), noiselevel=-1) + else: + for alias in aliases: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag in '%s': '%s'\n") % + (real_flag, file_name, alias), noiselevel=-1) + else: + if any(alias in v for k, v in file_dict.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias in '%s': '%s'\n") % + (file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _parse_repository_packageusealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "package.use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + lines = grabfile(file_name, recursive=True) + file_dict = {} + for line in lines: + elements = line.split() + atom = elements[0] + try: + atom = Atom(atom, eapi=eapi) + except InvalidAtom: + writemsg(_("--- Invalid atom in '%s': '%s'\n") % (file_name, atom)) + continue + if len(elements) == 1: + writemsg(_("--- Missing real USE flag for '%s' in '%s'\n") % (atom, file_name), noiselevel=-1) + continue + real_flag = elements[1] + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag for '%s' in '%s': '%s'\n") % (atom, file_name, real_flag), noiselevel=-1) + else: + for alias in elements[2:]: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag for '%s' in '%s': '%s'\n") % + (real_flag, atom, file_name, alias), noiselevel=-1) + else: + # Duplicated USE flag aliases in entries for different atoms + # matching the same package version are detected in getUseAliases(). + if any(alias in v for k, v in file_dict.get(atom.cp, {}).get(atom, {}).items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s' in '%s': '%s'\n") % + (atom, file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(atom.cp, {}).setdefault(atom, {}).setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _isStable(self, pkg): + if self._user_config: + try: + return pkg.stable + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + try: + pkg._metadata + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + # Since repoman uses different config instances for + # different profiles, we have to be careful to do the + # stable check against the correct profile here. + return self._is_stable(pkg) + + def getUseMask(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._usemask_list, incremental=True)) @@ -155,7 +303,12 @@ class UseManager(object): repo = dep_getrepo(pkg) pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + usemask = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -166,30 +319,56 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: usemask.append(self._repo_usemask_dict.get(repo, {})) + if stable: + usemask.append(self._repo_usestablemask_dict.get(repo, {})) cpdict = self._repo_pusemask_dict.get(repo, {}).get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._repo_pusestablemask_dict.get(repo, {}).get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + for i, pusemask_dict in enumerate(self._pusemask_list): if self._usemask_list[i]: usemask.append(self._usemask_list[i]) + if stable and self._usestablemask_list[i]: + usemask.append(self._usestablemask_list[i]) cpdict = pusemask_dict.get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._pusestablemask_list[i].get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + return frozenset(stack_lists(usemask, incremental=True)) - def getUseForce(self, pkg=None): + def getUseForce(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._useforce_list, incremental=True)) cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + useforce = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -200,25 +379,90 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: useforce.append(self._repo_useforce_dict.get(repo, {})) + if stable: + useforce.append(self._repo_usestableforce_dict.get(repo, {})) cpdict = self._repo_puseforce_dict.get(repo, {}).get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._repo_pusestableforce_dict.get(repo, {}).get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + for i, puseforce_dict in enumerate(self._puseforce_list): if self._useforce_list[i]: useforce.append(self._useforce_list[i]) + if stable and self._usestableforce_list[i]: + useforce.append(self._usestableforce_list[i]) cpdict = puseforce_dict.get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._pusestableforce_list[i].get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + return frozenset(stack_lists(useforce, incremental=True)) + def getUseAliases(self, pkg): + if hasattr(pkg, "eapi") and not eapi_has_use_aliases(pkg.eapi): + return {} + + cp = getattr(pkg, "cp", None) + if cp is None: + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + usealiases = {} + + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: + repos = [] + try: + repos.extend(repo.name for repo in + self.repositories[pkg.repo].masters) + except KeyError: + pass + repos.append(pkg.repo) + for repo in repos: + usealiases_dict = self._repo_usealiases_dict.get(repo, {}) + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + cp_usealiases_dict = self._repo_pusealiases_dict.get(repo, {}).get(cp) + if cp_usealiases_dict: + usealiases_dict_list = ordered_by_atom_specificity(cp_usealiases_dict, pkg) + for usealiases_dict in usealiases_dict_list: + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + + return usealiases + def getPUSE(self, pkg): cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp ret = "" cpdict = self._pusedict.get(cp) if cpdict: diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo Binary files differindex 2c9a609..16fa78e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo Binary files differindex b2ebd21..88010cc 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo Binary files differindex b03cc29..f3c8238 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo Binary files differindex aeee789..d78d7d2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo Binary files differindex 9854444..ef59bc0 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo Binary files differindex f2b9261..d2a012f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py index 1a75de9..f241e65 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'case_insensitive_vars', 'default_globals', 'env_blacklist', \ 'environ_filter', 'environ_whitelist', 'environ_whitelist_re', @@ -13,14 +15,17 @@ import re # configuration files. env_blacklist = frozenset(( "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", - "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_SKIP_MANIFEST", + "EBUILD_FORCE_TEST", "EBUILD_PHASE", + "EBUILD_PHASE_FUNC", "EBUILD_SKIP_MANIFEST", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", - "GREP_OPTIONS", "HOMEPAGE", "INHERITED", "IUSE", + "GREP_OPTIONS", "HDEPEND", "HOMEPAGE", + "INHERITED", "IUSE", "IUSE_EFFECTIVE", "KEYWORDS", "LICENSE", "MERGE_TYPE", "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND", - "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDIR_LOCKED", - "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", - "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", + "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_IUSE", + "PORTAGE_NONFATAL", "PORTAGE_PIPE_FD", "PORTAGE_REPO_NAME", "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY", "RESTRICT", "ROOT", "SLOT", "SRC_URI" )) @@ -39,7 +44,7 @@ environ_whitelist += [ "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "COLUMNS", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", - "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "ED", + "EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", "FEATURES", "FILESDIR", "HOME", "MERGE_TYPE", "NOCOLOR", "PATH", "PKGDIR", @@ -49,19 +54,26 @@ environ_whitelist += [ "PORTAGE_BINPKG_FILE", "PORTAGE_BINPKG_TAR_OPTS", "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", - "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", + "PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER", + "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS", + "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", + "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_INST_GID", "PORTAGE_INST_UID", - "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", - "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", - "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET", - "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", + "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", "PORTAGE_ECLASS_LOCATIONS", + "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PIPE_FD", + "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", + "PORTAGE_PYTHONPATH", "PORTAGE_QUIET", + "PORTAGE_REPO_NAME", "PORTAGE_REPOSITORIES", "PORTAGE_RESTRICT", "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", - "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", + "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", @@ -102,7 +114,7 @@ environ_whitelist += [ environ_whitelist += [ "CVS_RSH", "ECHANGELOG_USER", "GPG_AGENT_INFO", "LOG_SOCKET", - "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "SSH_AGENT_PID", "SSH_AUTH_SOCK", "STY", "WINDOW", "XAUTHORITY", ] @@ -133,9 +145,11 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ - "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", + "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", + "ACCEPT_RESTRICT", "AUTOCLEAN", "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "DCO_SIGNED_OFF_BY", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", @@ -144,8 +158,9 @@ environ_filter += [ "FETCHCOMMAND_RSYNC", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST", - "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", + "PORTAGE_BINHOST", "PORTAGE_BINPKG_FORMAT", + "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_CHECKSUM_FILTER", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", @@ -157,13 +172,20 @@ environ_filter += [ "PORTAGE_REPO_DUPLICATE_WARN", "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", - "PORTAGE_RSYNC_RETRIES", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", + "PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", + "PORTAGE_USE", + "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", - "SYNC", "USE_EXPAND_HIDDEN", "USE_ORDER", + "UNINSTALL_IGNORE", "USE_EXPAND_HIDDEN", "USE_ORDER", + "__PORTAGE_HELPER" +] + +# No longer supported variables +environ_filter += [ + "SYNC" ] environ_filter = frozenset(environ_filter) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo Binary files differindex 06ea37e..92c5d32 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py deleted file mode 100644 index d23677d..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import textwrap - -import portage -from portage.dep import _repo_separator -from portage.elog import elog_process -from portage.elog.messages import eerror - -def eapi_invalid(self, cpv, repo_name, settings, - eapi_var, eapi_parsed, eapi_lineno): - - msg = [] - msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" - " conform with PMS section 7.3.1 (see bug #402167):") % - (cpv, _repo_separator, repo_name), 70)) - - if not eapi_parsed: - # None means the assignment was not found, while an - # empty string indicates an (invalid) empty assingment. - msg.append( - "\tvalid EAPI assignment must" - " occur on or before line: %s" % - eapi_lineno) - else: - msg.append(("\tbash returned EAPI '%s' which does not match " - "assignment on line: %s") % - (eapi_var, eapi_lineno)) - - if 'parse-eapi-ebuild-head' in settings.features: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage," - " but at this time, it can by made non-fatal by setting" - " FEATURES=-parse-eapi-ebuild-head in" - " make.conf."), 70)) - else: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage." - " At the earliest opportunity, please enable" - " FEATURES=parse-eapi-ebuild-head in make.conf in order to" - " make this error fatal."), 70)) - - if portage.data.secpass >= 2: - # TODO: improve elog permission error handling (bug #416231) - for line in msg: - eerror(line, phase="other", key=cpv) - elog_process(cpv, settings, - phasefilter=("other",)) - - else: - out = portage.output.EOutput() - for line in msg: - out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo Binary files differdeleted file mode 100644 index 0181c03..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo Binary files differindex 315cb0f..744bb83 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo Binary files differindex 9f75518..ae66c3e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py index 7bbb0e8..351c956 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,12 +1,13 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import portage from portage import os -from portage import _unicode_decode -from portage.dep import Atom +from portage.dep import Atom, _repo_name_re from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom @@ -20,6 +21,12 @@ class QueryCommand(IpcCommand): _db = None + @classmethod + def get_db(cls): + if cls._db is not None: + return cls._db + return portage.db + def __init__(self, settings, phase): IpcCommand.__init__(self) self.settings = settings @@ -30,41 +37,46 @@ class QueryCommand(IpcCommand): @return: tuple of (stdout, stderr, returncode) """ - cmd, root, atom_str = argv + # Python 3: + # cmd, root, *args = argv + cmd = argv[0] + root = argv[1] + args = argv[2:] + + warnings = [] + warnings_str = '' + db = self.get_db() eapi = self.settings.get('EAPI') - allow_repo = eapi_has_repo_deps(eapi) - try: - atom = Atom(atom_str, allow_repo=allow_repo) - except InvalidAtom: - return ('', 'invalid atom: %s\n' % atom_str, 2) - warnings = [] - try: - atom = Atom(atom_str, allow_repo=allow_repo, eapi=eapi) - except InvalidAtom as e: - warnings.append(_unicode_decode("QA Notice: %s: %s") % (cmd, e)) + root = normalize_path(root).rstrip(os.path.sep) + os.path.sep + if root not in db: + return ('', '%s: Invalid ROOT: %s\n' % (cmd, root), 3) - use = self.settings.get('PORTAGE_BUILT_USE') - if use is None: - use = self.settings['PORTAGE_USE'] + portdb = db[root]["porttree"].dbapi + vardb = db[root]["vartree"].dbapi - use = frozenset(use.split()) - atom = atom.evaluate_conditionals(use) + if cmd in ('best_version', 'has_version'): + allow_repo = eapi_has_repo_deps(eapi) + try: + atom = Atom(args[0], allow_repo=allow_repo) + except InvalidAtom: + return ('', '%s: Invalid atom: %s\n' % (cmd, args[0]), 2) - db = self._db - if db is None: - db = portage.db + try: + atom = Atom(args[0], allow_repo=allow_repo, eapi=eapi) + except InvalidAtom as e: + warnings.append("QA Notice: %s: %s" % (cmd, e)) - warnings_str = '' - if warnings: - warnings_str = self._elog('eqawarn', warnings) + use = self.settings.get('PORTAGE_BUILT_USE') + if use is None: + use = self.settings['PORTAGE_USE'] - root = normalize_path(root).rstrip(os.path.sep) + os.path.sep - if root not in db: - return ('', 'invalid ROOT: %s\n' % root, 2) + use = frozenset(use.split()) + atom = atom.evaluate_conditionals(use) - vardb = db[root]["vartree"].dbapi + if warnings: + warnings_str = self._elog('eqawarn', warnings) if cmd == 'has_version': if vardb.match(atom): @@ -75,8 +87,35 @@ class QueryCommand(IpcCommand): elif cmd == 'best_version': m = best(vardb.match(atom)) return ('%s\n' % m, warnings_str, 0) + elif cmd in ('master_repositories', 'repository_path', 'available_eclasses', 'eclass_path', 'license_path'): + repo = _repo_name_re.match(args[0]) + if repo is None: + return ('', '%s: Invalid repository: %s\n' % (cmd, args[0]), 2) + try: + repo = portdb.repositories[args[0]] + except KeyError: + return ('', warnings_str, 1) + + if cmd == 'master_repositories': + return ('%s\n' % ' '.join(x.name for x in repo.masters), warnings_str, 0) + elif cmd == 'repository_path': + return ('%s\n' % repo.location, warnings_str, 0) + elif cmd == 'available_eclasses': + return ('%s\n' % ' '.join(sorted(repo.eclass_db.eclasses)), warnings_str, 0) + elif cmd == 'eclass_path': + try: + eclass = repo.eclass_db.eclasses[args[1]] + except KeyError: + return ('', warnings_str, 1) + return ('%s\n' % eclass.location, warnings_str, 0) + elif cmd == 'license_path': + paths = reversed([os.path.join(x.location, 'licenses', args[1]) for x in list(repo.masters) + [repo]]) + for path in paths: + if os.path.exists(path): + return ('%s\n' % path, warnings_str, 0) + return ('', warnings_str, 1) else: - return ('', 'invalid command: %s\n' % cmd, 2) + return ('', 'Invalid command: %s\n' % cmd, 3) def _elog(self, elog_funcname, lines): """ diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo Binary files differindex 0e9ee96..3087272 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo Binary files differindex d9f8d25..270e321 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py index 94f8c79..0fc53c8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py @@ -1,8 +1,9 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import tempfile +import portage from portage import os from portage import shutil from portage.const import EBUILD_PHASES @@ -10,10 +11,12 @@ from portage.elog import elog_process from portage.package.ebuild.config import config from portage.package.ebuild.doebuild import doebuild_environment from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildPhase import EbuildPhase -from _emerge.PollScheduler import PollScheduler -def spawn_nofetch(portdb, ebuild_path, settings=None): +def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): """ This spawns pkg_nofetch if appropriate. The settings parameter is useful only if setcpv has already been called in order @@ -47,7 +50,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings = config(clone=settings) if 'PORTAGE_PARALLEL_FETCHONLY' in settings: - return + return os.EX_OK # We must create our private PORTAGE_TMPDIR before calling # doebuild_environment(), since lots of variables such @@ -59,7 +62,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings['PORTAGE_TMPDIR'] = private_tmpdir settings.backup_changes('PORTAGE_TMPDIR') # private temp dir was just created, so it's not locked yet - settings.pop('PORTAGE_BUILDIR_LOCKED', None) + settings.pop('PORTAGE_BUILDDIR_LOCKED', None) try: doebuild_environment(ebuild_path, 'nofetch', @@ -73,14 +76,18 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): if 'fetch' not in restrict and \ 'nofetch' not in defined_phases: - return + return os.EX_OK prepare_build_dirs(settings=settings) ebuild_phase = EbuildPhase(background=False, - phase='nofetch', scheduler=PollScheduler().sched_iface, - settings=settings) + phase='nofetch', + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + fd_pipes=fd_pipes, settings=settings) ebuild_phase.start() ebuild_phase.wait() elog_process(settings.mycpv, settings) finally: shutil.rmtree(private_tmpdir) + + return ebuild_phase.returncode diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo Binary files differindex ac449ea..d4c597c 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.py b/portage_with_autodep/pym/portage/package/ebuild/config.py index 97cbd99..92e6c3f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.py +++ b/portage_with_autodep/pym/portage/package/ebuild/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'autouse', 'best_from_dict', 'check_config_instance', 'config', ] @@ -19,6 +21,8 @@ from _emerge.Package import Package import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid', + 'portage.dbapi.vartree:vartree', + 'portage.package.ebuild.doebuild:_phase_func_map', ) from portage import bsd_chflags, \ load_mod, os, selinux, _unicode_decode @@ -29,10 +33,9 @@ from portage.const import CACHE_PATH, \ USER_VIRTUALS_FILE from portage.dbapi import dbapi from portage.dbapi.porttree import portdbapi -from portage.dbapi.vartree import vartree from portage.dep import Atom, isvalidatom, match_from_list, use_reduce, _repo_separator, _slot_separator from portage.eapi import eapi_exports_AA, eapi_exports_merge_type, \ - eapi_supports_prefix, eapi_exports_replace_vars + eapi_supports_prefix, eapi_exports_replace_vars, _get_eapi_attrs from portage.env.loaders import KeyValuePairFileLoader from portage.exception import InvalidDependString, PortageException from portage.localization import _ @@ -42,7 +45,8 @@ from portage.repository.config import load_repository_config from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ - writemsg, writemsg_level + writemsg, writemsg_level, _eapi_cache +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars @@ -55,10 +59,29 @@ from portage.package.ebuild._config.LocationsManager import LocationsManager from portage.package.ebuild._config.MaskManager import MaskManager from portage.package.ebuild._config.VirtualsManager import VirtualsManager from portage.package.ebuild._config.helper import ordered_by_atom_specificity, prune_incremental +from portage.package.ebuild._config.unpack_dependencies import load_unpack_dependencies_configuration if sys.hexversion >= 0x3000000: basestring = str +_feature_flags_cache = {} + +def _get_feature_flags(eapi_attrs): + cache_key = (eapi_attrs.feature_flag_test, eapi_attrs.feature_flag_targetroot) + flags = _feature_flags_cache.get(cache_key) + if flags is not None: + return flags + + flags = [] + if eapi_attrs.feature_flag_test: + flags.append("test") + if eapi_attrs.feature_flag_targetroot: + flags.append("targetroot") + + flags = frozenset(flags) + _feature_flags_cache[cache_key] = flags + return flags + def autouse(myvartree, use_cache=1, mysettings=None): warnings.warn("portage.autouse() is deprecated", DeprecationWarning, stacklevel=2) @@ -123,9 +146,9 @@ class config(object): """ _constant_keys = frozenset(['PORTAGE_BIN_PATH', 'PORTAGE_GID', - 'PORTAGE_PYM_PATH']) + 'PORTAGE_PYM_PATH', 'PORTAGE_PYTHONPATH']) - _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', + _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'HDEPEND', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT', 'repository', 'RESTRICT', 'LICENSE',) @@ -146,7 +169,7 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, eprefix=None, local_config=True, env=None, - _unmatched_removal=False): + _unmatched_removal=False, repositories=None): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -160,7 +183,8 @@ class config(object): @type config_incrementals: List @param config_root: path to read local config from (defaults to "/", see PORTAGE_CONFIGROOT) @type config_root: String - @param target_root: __init__ override of $ROOT env variable. + @param target_root: the target root, which typically corresponds to the + value of the $ROOT env variable (default is /) @type target_root: String @param eprefix: set the EPREFIX variable (default is portage.const.EPREFIX) @type eprefix: String @@ -173,14 +197,21 @@ class config(object): @param _unmatched_removal: Enabled by repoman when the --unmatched-removal option is given. @type _unmatched_removal: Boolean + @param repositories: Configuration of repositories. + Defaults to portage.repository.config.load_repository_config(). + @type repositories: Instance of portage.repository.config.RepoConfigLoader class. """ + # This is important when config is reloaded after emerge --sync. + _eapi_cache.clear() + # When initializing the global portage.settings instance, avoid # raising exceptions whenever possible since exceptions thrown # from 'import portage' or 'import portage.exceptions' statements # can practically render the api unusable for api consumers. tolerant = hasattr(portage, '_initializing_globals') self._tolerant = tolerant + self._unmatched_removal = _unmatched_removal self.locked = 0 self.mycpv = None @@ -191,8 +222,10 @@ class config(object): self.uvlist = [] self._accept_chost_re = None self._accept_properties = None + self._accept_restrict = None self._features_overrides = [] self._make_defaults = None + self._parent_stable = None # _unknown_features records unknown features that # have triggered warning messages, and ensures that @@ -205,6 +238,7 @@ class config(object): # For immutable attributes, use shallow copy for # speed and memory conservation. self._tolerant = clone._tolerant + self._unmatched_removal = clone._unmatched_removal self.categories = clone.categories self.depcachedir = clone.depcachedir self.incrementals = clone.incrementals @@ -213,6 +247,8 @@ class config(object): self.profiles = clone.profiles self.packages = clone.packages self.repositories = clone.repositories + self.unpack_dependencies = clone.unpack_dependencies + self._iuse_effective = clone._iuse_effective self._iuse_implicit_match = clone._iuse_implicit_match self._non_user_variables = clone._non_user_variables self._env_d_blacklist = clone._env_d_blacklist @@ -227,9 +263,12 @@ class config(object): self._setcpv_args_hash = clone._setcpv_args_hash # immutable attributes (internal policy ensures lack of mutation) - self._keywords_manager = clone._keywords_manager + self._locations_manager = clone._locations_manager self._use_manager = clone._use_manager - self._mask_manager = clone._mask_manager + # force instantiation of lazy immutable objects when cloning, so + # that they're not instantiated more than once + self._keywords_manager_obj = clone._keywords_manager + self._mask_manager_obj = clone._mask_manager # shared mutable attributes self._unknown_features = clone._unknown_features @@ -266,30 +305,55 @@ class config(object): #all LicenseManager instances. self._license_manager = clone._license_manager - self._virtuals_manager = copy.deepcopy(clone._virtuals_manager) + # force instantiation of lazy objects when cloning, so + # that they're not instantiated more than once + self._virtuals_manager_obj = copy.deepcopy(clone._virtuals_manager) self._accept_properties = copy.deepcopy(clone._accept_properties) self._ppropertiesdict = copy.deepcopy(clone._ppropertiesdict) + self._accept_restrict = copy.deepcopy(clone._accept_restrict) + self._paccept_restrict = copy.deepcopy(clone._paccept_restrict) self._penvdict = copy.deepcopy(clone._penvdict) self._expand_map = copy.deepcopy(clone._expand_map) else: + # lazily instantiated objects + self._keywords_manager_obj = None + self._mask_manager_obj = None + self._virtuals_manager_obj = None + locations_manager = LocationsManager(config_root=config_root, config_profile_path=config_profile_path, eprefix=eprefix, local_config=local_config, target_root=target_root) + self._locations_manager = locations_manager eprefix = locations_manager.eprefix config_root = locations_manager.config_root abs_user_config = locations_manager.abs_user_config + make_conf_paths = [ + os.path.join(config_root, 'etc', 'make.conf'), + os.path.join(config_root, MAKE_CONF_FILE) + ] + try: + if os.path.samefile(*make_conf_paths): + make_conf_paths.pop() + except OSError: + pass - make_conf = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True) or {} - - make_conf.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=make_conf) or {}) + make_conf_count = 0 + make_conf = {} + for x in make_conf_paths: + mygcfg = getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=make_conf, recursive=True) + if mygcfg is not None: + make_conf.update(mygcfg) + make_conf_count += 1 + + if make_conf_count == 2: + writemsg("!!! %s\n" % + _("Found 2 make.conf files, using both '%s' and '%s'") % + tuple(make_conf_paths), noiselevel=-1) # Allow ROOT setting to come from make.conf if it's not overridden # by the constructor argument (from the calling environment). @@ -315,15 +379,30 @@ class config(object): # lead to unexpected results. env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=False) or {} + tolerant=tolerant, expand=False) or {} expand_map = env_d.copy() self._expand_map = expand_map # Allow make.globals to set default paths relative to ${EPREFIX}. expand_map["EPREFIX"] = eprefix - make_globals = getconfig(os.path.join( - self.global_config_path, 'make.globals'), expand=expand_map) + make_globals_path = os.path.join( + self.global_config_path, 'make.globals') + old_make_globals = os.path.join(config_root, + 'etc', 'make.globals') + if os.path.isfile(old_make_globals) and \ + not os.path.samefile(make_globals_path, old_make_globals): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found obsolete make.globals file: " + "'%s', (using '%s' instead)") % + (old_make_globals, make_globals_path), + noiselevel=-1) + + make_globals = getconfig(make_globals_path, + tolerant=tolerant, expand=expand_map) if make_globals is None: make_globals = {} @@ -412,6 +491,7 @@ class config(object): known_repos = [] portdir = "" portdir_overlay = "" + portdir_sync = None for confs in [make_globals, make_conf, self.configdict["env"]]: v = confs.get("PORTDIR") if v is not None: @@ -421,12 +501,52 @@ class config(object): if v is not None: portdir_overlay = v known_repos.extend(shlex_split(v)) + v = confs.get("SYNC") + if v is not None: + portdir_sync = v + known_repos = frozenset(known_repos) self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay + if portdir_sync: + self["SYNC"] = portdir_sync self.lookuplist = [self.configdict["env"]] - self.repositories = load_repository_config(self) + if repositories is None: + self.repositories = load_repository_config(self) + else: + self.repositories = repositories + + self['PORTAGE_REPOSITORIES'] = self.repositories.config_string() + self.backup_changes('PORTAGE_REPOSITORIES') + + #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility + main_repo = self.repositories.mainRepo() + if main_repo is not None: + self["PORTDIR"] = main_repo.user_location + self.backup_changes("PORTDIR") + expand_map["PORTDIR"] = self["PORTDIR"] + + # repoman controls PORTDIR_OVERLAY via the environment, so no + # special cases are needed here. + portdir_overlay = list(self.repositories.repoUserLocationList()) + if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: + portdir_overlay = portdir_overlay[1:] + + new_ov = [] + if portdir_overlay: + for ov in portdir_overlay: + ov = normalize_path(ov) + if isdir_raise_eaccess(ov) or portage._sync_disabled_warnings: + new_ov.append(portage._shell_quote(ov)) + else: + writemsg(_("!!! Invalid PORTDIR_OVERLAY" + " (not a dir): '%s'\n") % ov, noiselevel=-1) + self["PORTDIR_OVERLAY"] = " ".join(new_ov) + self.backup_changes("PORTDIR_OVERLAY") + expand_map["PORTDIR_OVERLAY"] = self["PORTDIR_OVERLAY"] + + locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) locations_manager.load_profiles(self.repositories, known_repos) profiles_complex = locations_manager.profiles_complex @@ -446,11 +566,13 @@ class config(object): x = Atom(x.lstrip('*')) self.prevmaskdict.setdefault(x.cp, []).append(x) + self.unpack_dependencies = load_unpack_dependencies_configuration(self.repositories) mygcfg = {} - if self.profiles: - mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), - expand=expand_map) for x in self.profiles] + if profiles_complex: + mygcfg_dlists = [getconfig(os.path.join(x.location, "make.defaults"), + tolerant=tolerant, expand=expand_map, recursive=x.portage1_directories) + for x in profiles_complex] self._make_defaults = mygcfg_dlists mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) @@ -459,15 +581,11 @@ class config(object): self.configlist.append(mygcfg) self.configdict["defaults"]=self.configlist[-1] - mygcfg = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {} - - mygcfg.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {}) + mygcfg = {} + for x in make_conf_paths: + mygcfg.update(getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=expand_map, recursive=True) or {}) # Don't allow the user to override certain variables in make.conf profile_only_variables = self.configdict["defaults"].get( @@ -520,66 +638,34 @@ class config(object): self.backup_changes("PORTAGE_CONFIGROOT") self["ROOT"] = target_root self.backup_changes("ROOT") - - # The PORTAGE_OVERRIDE_EPREFIX variable propagates the EPREFIX - # of this config instance to any portage commands or API - # consumers running in subprocesses. self["EPREFIX"] = eprefix self.backup_changes("EPREFIX") - self["PORTAGE_OVERRIDE_EPREFIX"] = eprefix - self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") self["EROOT"] = eroot self.backup_changes("EROOT") + # The prefix of the running portage instance is used in the + # ebuild environment to implement the --host-root option for + # best_version and has_version. + self["PORTAGE_OVERRIDE_EPREFIX"] = portage.const.EPREFIX + self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") + self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) + self._paccept_restrict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility - main_repo = self.repositories.mainRepo() - if main_repo is not None: - self["PORTDIR"] = main_repo.user_location - self.backup_changes("PORTDIR") - - # repoman controls PORTDIR_OVERLAY via the environment, so no - # special cases are needed here. - portdir_overlay = list(self.repositories.repoUserLocationList()) - if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: - portdir_overlay = portdir_overlay[1:] - - new_ov = [] - if portdir_overlay: - shell_quote_re = re.compile(r"[\s\\\"'$`]") - for ov in portdir_overlay: - ov = normalize_path(ov) - if os.path.isdir(ov): - if shell_quote_re.search(ov) is not None: - ov = portage._shell_quote(ov) - new_ov.append(ov) - else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) - - self["PORTDIR_OVERLAY"] = " ".join(new_ov) - self.backup_changes("PORTDIR_OVERLAY") - - locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) - self._repo_make_defaults = {} for repo in self.repositories.repos_with_profiles(): d = getconfig(os.path.join(repo.location, "profiles", "make.defaults"), - expand=self.configdict["globals"].copy()) or {} + tolerant=tolerant, expand=self.configdict["globals"].copy(), recursive=repo.portage1_profiles) or {} if d: for k in chain(self._env_blacklist, profile_only_variables, self._global_only_vars): d.pop(k, None) self._repo_make_defaults[repo.name] = d - #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ - local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) - #Read all USE related files from profiles and optionally from user config. - self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) + self._use_manager = UseManager(self.repositories, profiles_complex, + abs_user_config, self._isStable, user_config=local_config) #Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() @@ -595,13 +681,6 @@ class config(object): self._license_manager.extract_global_changes( \ self.configdict["conf"].get("ACCEPT_LICENSE", "")) - #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, profiles_complex, - abs_user_config, user_config=local_config, - strict_umatched_removal=_unmatched_removal) - - self._virtuals_manager = VirtualsManager(self.profiles) - if local_config: #package.properties propdict = grabdict_package(os.path.join( @@ -616,6 +695,20 @@ class config(object): for k, v in propdict.items(): self._ppropertiesdict.setdefault(k.cp, {})[k] = v + # package.accept_restrict + d = grabdict_package(os.path.join( + abs_user_config, "package.accept_restrict"), + recursive=True, allow_wildcard=True, + allow_repo=True, verify_eapi=False) + v = d.pop("*/*", None) + if v is not None: + if "ACCEPT_RESTRICT" in self.configdict["conf"]: + self.configdict["conf"]["ACCEPT_RESTRICT"] += " " + " ".join(v) + else: + self.configdict["conf"]["ACCEPT_RESTRICT"] = " ".join(v) + for k, v in d.items(): + self._paccept_restrict.setdefault(k.cp, {})[k] = v + #package.env penvdict = grabdict_package(os.path.join( abs_user_config, "package.env"), recursive=1, allow_wildcard=True, \ @@ -702,21 +795,9 @@ class config(object): self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal:repo:env.d" self.depcachedir = DEPCACHE_PATH - if eprefix: - # See comments about make.globals and EPREFIX - # above. DEPCACHE_PATH is similar. - if target_root == "/": - # case (1) above - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume DEPCACHE_PATH is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.depcachedir = os.path.join(portage.const.EPREFIX, + DEPCACHE_PATH.lstrip(os.sep)) if self.get("PORTAGE_DEPCACHEDIR", None): self.depcachedir = self["PORTAGE_DEPCACHEDIR"] @@ -783,12 +864,17 @@ class config(object): self[var] = default_val self.backup_changes(var) + if portage._internal_caller: + self["PORTAGE_INTERNAL_CALLER"] = "1" + self.backup_changes("PORTAGE_INTERNAL_CALLER") + # initialize self.features self.regenerate() if bsd_chflags: self.features.add('chflags') + self._iuse_effective = self._calc_iuse_effective() self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() @@ -798,11 +884,6 @@ class config(object): self[k] = self[k].lower() self.backup_changes(k) - if main_repo is not None and not main_repo.sync: - main_repo_sync = self.get("SYNC") - if main_repo_sync: - main_repo.sync = main_repo_sync - # The first constructed config object initializes these modules, # and subsequent calls to the _init() functions have no effect. portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) @@ -882,6 +963,32 @@ class config(object): noiselevel=-1) @property + def _keywords_manager(self): + if self._keywords_manager_obj is None: + self._keywords_manager_obj = KeywordsManager( + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + self.local_config, + global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) + return self._keywords_manager_obj + + @property + def _mask_manager(self): + if self._mask_manager_obj is None: + self._mask_manager_obj = MaskManager(self.repositories, + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + user_config=self.local_config, + strict_umatched_removal=self._unmatched_removal) + return self._mask_manager_obj + + @property + def _virtuals_manager(self): + if self._virtuals_manager_obj is None: + self._virtuals_manager_obj = VirtualsManager(self.profiles) + return self._virtuals_manager_obj + + @property def pkeywordsdict(self): result = self._keywords_manager.pkeywordsdict.copy() for k, v in result.items(): @@ -919,13 +1026,23 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - profile_broken = not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent")) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + profile_broken = False + + if not self.profile_path: + profile_broken = True + else: + # If any one of these files exists, then + # the profile is considered valid. + for x in ("make.defaults", "parent", + "packages", "use.force", "use.mask"): + if exists_raise_eaccess(os.path.join(self.profile_path, x)): + break + else: + profile_broken = True - if profile_broken: + if profile_broken and not portage._sync_disabled_warnings: abs_profile_path = None - for x in (PROFILE_PATH, 'etc/portage/make.profile'): + for x in (PROFILE_PATH, 'etc/make.profile'): x = os.path.join(self["PORTAGE_CONFIGROOT"], x) try: os.lstat(x) @@ -1091,8 +1208,11 @@ class config(object): the previously calculated USE settings. """ - def __init__(self, use, usemask, iuse_implicit, + def __init__(self, settings, unfiltered_use, + use, usemask, iuse_implicit, use_expand_split, use_expand_dict): + self._settings = settings + self._unfiltered_use = unfiltered_use self._use = use self._usemask = usemask self._iuse_implicit = iuse_implicit @@ -1147,13 +1267,32 @@ class config(object): # Don't export empty USE_EXPAND vars unless the user config # exports them as empty. This is required for vars such as # LINGUAS, where unset and empty have different meanings. + # The special '*' token is understood by ebuild.sh, which + # will unset the variable so that things like LINGUAS work + # properly (see bug #459350). if has_wildcard: - # ebuild.sh will see this and unset the variable so - # that things like LINGUAS work properly value = '*' else: if has_iuse: - value = '' + already_set = False + # Skip the first 'env' configdict, in order to + # avoid infinite recursion here, since that dict's + # __getitem__ calls the current __getitem__. + for d in self._settings.lookuplist[1:]: + if key in d: + already_set = True + break + + if not already_set: + for x in self._unfiltered_use: + if x[:prefix_len] == prefix: + already_set = True + break + + if already_set: + value = '' + else: + value = '*' else: # It's not in IUSE, so just allow the variable content # to pass through if it is defined somewhere. This @@ -1189,7 +1328,7 @@ class config(object): if not isinstance(mycpv, basestring): pkg = mycpv mycpv = pkg.cpv - mydb = pkg.metadata + mydb = pkg._metadata explicit_iuse = pkg.iuse.all args_hash = (mycpv, id(pkg)) if pkg.built: @@ -1210,6 +1349,7 @@ class config(object): iuse = "" pkg_configdict = self.configdict["pkg"] previous_iuse = pkg_configdict.get("IUSE") + previous_iuse_effective = pkg_configdict.get("IUSE_EFFECTIVE") previous_features = pkg_configdict.get("FEATURES") aux_keys = self._setcpv_aux_keys @@ -1221,6 +1361,7 @@ class config(object): pkg_configdict["CATEGORY"] = cat pkg_configdict["PF"] = pf repository = None + eapi = None if mydb: if not hasattr(mydb, "aux_get"): for k in aux_keys: @@ -1247,14 +1388,16 @@ class config(object): # Empty USE means this dbapi instance does not contain # built packages. built_use = None + eapi = pkg_configdict['EAPI'] repository = pkg_configdict.pop("repository", None) if repository is not None: pkg_configdict["PORTAGE_REPO_NAME"] = repository - slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) + self.mycpv = _pkg_str(self.mycpv, metadata=pkg_configdict, + settings=self) + cpv_slot = self.mycpv else: cpv_slot = pkg pkginternaluse = [] @@ -1264,6 +1407,9 @@ class config(object): elif x.startswith("-"): pkginternaluse.append(x) pkginternaluse = " ".join(pkginternaluse) + + eapi_attrs = _get_eapi_attrs(eapi) + if pkginternaluse != self.configdict["pkginternal"].get("USE", ""): self.configdict["pkginternal"]["USE"] = pkginternaluse has_changed = True @@ -1394,30 +1540,70 @@ class config(object): # If reset() has not been called, it's safe to return # early if IUSE has not changed. - if not has_changed and previous_iuse == iuse: + if not has_changed and previous_iuse == iuse and \ + (previous_iuse_effective is not None == eapi_attrs.iuse_effective): return # Filter out USE flags that aren't part of IUSE. This has to # be done for every setcpv() call since practically every # package has different IUSE. use = set(self["USE"].split()) + unfiltered_use = frozenset(use) if explicit_iuse is None: explicit_iuse = frozenset(x.lstrip("+-") for x in iuse.split()) - iuse_implicit_match = self._iuse_implicit_match - portage_iuse = self._get_implicit_iuse() - portage_iuse.update(explicit_iuse) + + if eapi_attrs.iuse_effective: + iuse_implicit_match = self._iuse_effective_match + portage_iuse = set(self._iuse_effective) + portage_iuse.update(explicit_iuse) + self.configdict["pkg"]["IUSE_EFFECTIVE"] = \ + " ".join(sorted(portage_iuse)) + else: + iuse_implicit_match = self._iuse_implicit_match + portage_iuse = self._get_implicit_iuse() + portage_iuse.update(explicit_iuse) # PORTAGE_IUSE is not always needed so it's lazily evaluated. self.configdict["env"].addLazySingleton( "PORTAGE_IUSE", _lazy_iuse_regex, portage_iuse) - ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" + if pkg is None: + raw_restrict = pkg_configdict.get("RESTRICT") + else: + raw_restrict = pkg._raw_metadata["RESTRICT"] + + restrict_test = False + if raw_restrict: + try: + if built_use is not None: + restrict = use_reduce(raw_restrict, + uselist=built_use, flat=True) + else: + # Use matchnone=True to ignore USE conditional parts + # of RESTRICT, since we want to know whether to mask + # the "test" flag _before_ we know the USE values + # that would be needed to evaluate the USE + # conditionals (see bug #273272). + restrict = use_reduce(raw_restrict, + matchnone=True, flat=True) + except PortageException: + pass + else: + restrict_test = "test" in restrict + + ebuild_force_test = not restrict_test and \ + self.get("EBUILD_FORCE_TEST") == "1" + if ebuild_force_test and \ not hasattr(self, "_ebuild_force_test_msg_shown"): self._ebuild_force_test_msg_shown = True writemsg(_("Forcing test.\n"), noiselevel=-1) - if "test" in self.features: - if "test" in self.usemask and not ebuild_force_test: + + if "test" in explicit_iuse or iuse_implicit_match("test"): + if "test" not in self.features: + use.discard("test") + elif restrict_test or \ + ("test" in self.usemask and not ebuild_force_test): # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. @@ -1430,6 +1616,13 @@ class config(object): self.usemask = \ frozenset(x for x in self.usemask if x != "test") + if eapi_attrs.feature_flag_targetroot and \ + ("targetroot" in explicit_iuse or iuse_implicit_match("targetroot")): + if self["ROOT"] != "/": + use.add("targetroot") + else: + use.discard("targetroot") + # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update([x for x in use \ if (x not in explicit_iuse and \ @@ -1440,7 +1633,8 @@ class config(object): # comparison instead of startswith(). use_expand_split = set(x.lower() for \ x in self.get('USE_EXPAND', '').split()) - lazy_use_expand = self._lazy_use_expand(use, self.usemask, + lazy_use_expand = self._lazy_use_expand( + self, unfiltered_use, use, self.usemask, portage_iuse, use_expand_split, self._use_expand_dict) use_expand_iuses = {} @@ -1470,6 +1664,14 @@ class config(object): self.configdict['env'].addLazySingleton(k, lazy_use_expand.__getitem__, k) + for k in self.get("USE_EXPAND_UNPREFIXED", "").split(): + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in use ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + # Filtered for the ebuild environment. Store this in a separate # attribute since we still want to be able to see global USE # settings for things like emerge --info. @@ -1477,6 +1679,10 @@ class config(object): self.configdict["env"]["PORTAGE_USE"] = \ " ".join(sorted(x for x in use if x[-2:] != '_*')) + # Clear the eapi cache here rather than in the constructor, since + # setcpv triggers lazy instantiation of things like _use_manager. + _eapi_cache.clear() + def _grab_pkg_env(self, penv, container, protected_keys=None): if protected_keys is None: protected_keys = () @@ -1510,9 +1716,42 @@ class config(object): else: container[k] = v + def _iuse_effective_match(self, flag): + return flag in self._iuse_effective + + def _calc_iuse_effective(self): + """ + Beginning with EAPI 5, IUSE_EFFECTIVE is defined by PMS. + """ + iuse_effective = [] + iuse_effective.extend(self.get("IUSE_IMPLICIT", "").split()) + + # USE_EXPAND_IMPLICIT should contain things like ARCH, ELIBC, + # KERNEL, and USERLAND. + use_expand_implicit = frozenset( + self.get("USE_EXPAND_IMPLICIT", "").split()) + + # USE_EXPAND_UNPREFIXED should contain at least ARCH, and + # USE_EXPAND_VALUES_ARCH should contain all valid ARCH flags. + for v in self.get("USE_EXPAND_UNPREFIXED", "").split(): + if v not in use_expand_implicit: + continue + iuse_effective.extend( + self.get("USE_EXPAND_VALUES_" + v, "").split()) + + use_expand = frozenset(self.get("USE_EXPAND", "").split()) + for v in use_expand_implicit: + if v not in use_expand: + continue + lower_v = v.lower() + for x in self.get("USE_EXPAND_VALUES_" + v, "").split(): + iuse_effective.append(lower_v + "_" + x) + + return frozenset(iuse_effective) + def _get_implicit_iuse(self): """ - Some flags are considered to + Prior to EAPI 5, these flags are considered to be implicit members of IUSE: * Flags derived from ARCH * Flags derived from USE_EXPAND_HIDDEN variables @@ -1549,11 +1788,11 @@ class config(object): return iuse_implicit - def _getUseMask(self, pkg): - return self._use_manager.getUseMask(pkg) + def _getUseMask(self, pkg, stable=None): + return self._use_manager.getUseMask(pkg, stable=stable) - def _getUseForce(self, pkg): - return self._use_manager.getUseForce(pkg) + def _getUseForce(self, pkg, stable=None): + return self._use_manager.getUseForce(pkg, stable=stable) def _getMaskAtom(self, cpv, metadata): """ @@ -1618,6 +1857,11 @@ class config(object): return x return None + def _isStable(self, pkg): + return self._keywords_manager.isStable(pkg, + self.get("ACCEPT_KEYWORDS", ""), + self.configdict["backupenv"].get("ACCEPT_KEYWORDS", "")) + def _getKeywords(self, cpv, metadata): return self._keywords_manager.getKeywords(cpv, metadata["SLOT"], \ metadata.get("KEYWORDS", ""), metadata.get("repository")) @@ -1706,9 +1950,10 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties - if not hasattr(cpv, 'slot'): - cpv = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: @@ -1720,7 +1965,6 @@ class config(object): properties_str = metadata.get("PROPERTIES", "") properties = set(use_reduce(properties_str, matchall=1, flat=True)) - properties.discard('||') acceptable_properties = set() for x in accept_properties: @@ -1738,40 +1982,58 @@ class config(object): else: use = [] - properties_struct = use_reduce(properties_str, uselist=use, opconvert=True) - return self._getMaskedProperties(properties_struct, acceptable_properties) - - def _getMaskedProperties(self, properties_struct, acceptable_properties): - if not properties_struct: - return [] - if properties_struct[0] == "||": - ret = [] - for element in properties_struct[1:]: - if isinstance(element, list): - if element: - tmp = self._getMaskedProperties( - element, acceptable_properties) - if not tmp: - return [] - ret.extend(tmp) - else: - if element in acceptable_properties: - return[] - ret.append(element) - # Return all masked properties, since we don't know which combination - # (if any) the user will decide to unmask - return ret - - ret = [] - for element in properties_struct: - if isinstance(element, list): - if element: - ret.extend(self._getMaskedProperties(element, - acceptable_properties)) + return [x for x in use_reduce(properties_str, uselist=use, flat=True) + if x not in acceptable_properties] + + def _getMissingRestrict(self, cpv, metadata): + """ + Take a RESTRICT string and return a list of any tokens the user + may need to accept for the given package. The returned list will not + contain any tokens that have already been accepted. This method + can throw an InvalidDependString exception. + + @param cpv: The package name (for package.accept_restrict support) + @type cpv: String + @param metadata: A dictionary of raw package metadata + @type metadata: dict + @rtype: List + @return: A list of tokens that have not been accepted. + """ + accept_restrict = self._accept_restrict + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) + cp = cpv_getkey(cpv) + cpdict = self._paccept_restrict.get(cp) + if cpdict: + paccept_restrict_list = ordered_by_atom_specificity(cpdict, cpv) + if paccept_restrict_list: + accept_restrict = list(self._accept_restrict) + for x in paccept_restrict_list: + accept_restrict.extend(x) + + restrict_str = metadata.get("RESTRICT", "") + all_restricts = set(use_reduce(restrict_str, matchall=1, flat=True)) + + acceptable_restricts = set() + for x in accept_restrict: + if x == '*': + acceptable_restricts.update(all_restricts) + elif x == '-*': + acceptable_restricts.clear() + elif x[:1] == '-': + acceptable_restricts.discard(x[1:]) else: - if element not in acceptable_properties: - ret.append(element) - return ret + acceptable_restricts.add(x) + + if "?" in restrict_str: + use = metadata["USE"].split() + else: + use = [] + + return [x for x in use_reduce(restrict_str, uselist=use, flat=True) + if x not in acceptable_restricts] def _accept_chost(self, cpv, metadata): """ @@ -1840,7 +2102,8 @@ class config(object): """Reload things like /etc/profile.env that can change during runtime.""" env_d_filename = os.path.join(self["EROOT"], "etc", "profile.env") self.configdict["env.d"].clear() - env_d = getconfig(env_d_filename, expand=False) + env_d = getconfig(env_d_filename, + tolerant=self._tolerant, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. for k in self._env_d_blacklist: @@ -1909,6 +2172,18 @@ class config(object): # repoman will accept any property self._accept_properties = ('*',) + if self.local_config: + mysplit = [] + for curdb in mydbs: + mysplit.extend(curdb.get('ACCEPT_RESTRICT', '').split()) + mysplit = prune_incremental(mysplit) + self.configlist[-1]['ACCEPT_RESTRICT'] = ' '.join(mysplit) + if tuple(mysplit) != self._accept_restrict: + self._accept_restrict = tuple(mysplit) + else: + # repoman will accept any property + self._accept_restrict = ('*',) + increment_lists = {} for k in myincrementals: incremental_list = [] @@ -1963,6 +2238,8 @@ class config(object): if v is not None: use_expand_dict[k] = v + use_expand_unprefixed = self.get("USE_EXPAND_UNPREFIXED", "").split() + # In order to best accomodate the long-standing practice of # setting default USE_EXPAND variables in the profile's # make.defaults, we translate these variables into their @@ -1976,6 +2253,12 @@ class config(object): continue use = cfg.get("USE", "") expand_use = [] + + for k in use_expand_unprefixed: + v = cfg.get(k) + if v is not None: + expand_use.extend(v.split()) + for k in use_expand_dict: v = cfg.get(k) if v is None: @@ -2013,6 +2296,17 @@ class config(object): iuse = [x.lstrip("+-") for x in iuse.split()] myflags = set() for curdb in self.uvlist: + + for k in use_expand_unprefixed: + v = curdb.get(k) + if v is None: + continue + for x in v.split(): + if x[:1] == "-": + myflags.discard(x[1:]) + else: + myflags.add(x) + cur_use_expand = [x for x in use_expand if x in curdb] mysplit = curdb.get("USE", "").split() if not mysplit and not cur_use_expand: @@ -2129,6 +2423,14 @@ class config(object): elif k in self: self.configlist[-1][k] = '' + for k in use_expand_unprefixed: + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in myflags ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + @property def virts_p(self): warnings.warn("portage config.virts_p attribute " + \ @@ -2189,8 +2491,22 @@ class config(object): elif mykey == "PORTAGE_PYM_PATH": return portage._pym_path + elif mykey == "PORTAGE_PYTHONPATH": + value = [x for x in \ + self.backupenv.get("PYTHONPATH", "").split(":") if x] + need_pym_path = True + if value: + try: + need_pym_path = not os.path.samefile(value[0], + portage._pym_path) + except OSError: + pass + if need_pym_path: + value.insert(0, portage._pym_path) + return ":".join(value) + elif mykey == "PORTAGE_GID": - return _unicode_decode(str(portage_gid)) + return "%s" % portage_gid for d in self.lookuplist: try: @@ -2277,6 +2593,7 @@ class config(object): environ_filter = self._environ_filter eapi = self.get('EAPI') + eapi_attrs = _get_eapi_attrs(eapi) phase = self.get('EBUILD_PHASE') filter_calling_env = False if self.mycpv is not None and \ @@ -2358,14 +2675,20 @@ class config(object): not eapi_exports_replace_vars(eapi): mydict.pop("REPLACED_BY_VERSION", None) + if phase is not None and eapi_attrs.exports_EBUILD_PHASE_FUNC: + phase_func = _phase_func_map.get(phase) + if phase_func is not None: + mydict["EBUILD_PHASE_FUNC"] = phase_func + return mydict def thirdpartymirrors(self): if getattr(self, "_thirdpartymirrors", None) is None: - profileroots = [os.path.join(self["PORTDIR"], "profiles")] - for x in shlex_split(self.get("PORTDIR_OVERLAY", "")): - profileroots.insert(0, os.path.join(x, "profiles")) - thirdparty_lists = [grabdict(os.path.join(x, "thirdpartymirrors")) for x in profileroots] + thirdparty_lists = [] + for repo_name in reversed(self.repositories.prepos_order): + thirdparty_lists.append(grabdict(os.path.join( + self.repositories[repo_name].location, + "profiles", "thirdpartymirrors"))) self._thirdpartymirrors = stack_dictlist(thirdparty_lists, incremental=True) return self._thirdpartymirrors diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.pyo b/portage_with_autodep/pym/portage/package/ebuild/config.pyo Binary files differindex 742ee2b..f4123ed 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/config.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py index 3fab4da..fdb19b4 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py @@ -1,10 +1,11 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['deprecated_profile_check'] import io +import portage from portage import os, _encodings, _unicode_encode from portage.const import DEPRECATED_PROFILE_FILE from portage.localization import _ @@ -12,16 +13,32 @@ from portage.output import colorize from portage.util import writemsg def deprecated_profile_check(settings=None): - config_root = "/" + config_root = None + eprefix = None + deprecated_profile_file = None if settings is not None: config_root = settings["PORTAGE_CONFIGROOT"] - deprecated_profile_file = os.path.join(config_root, - DEPRECATED_PROFILE_FILE) - if not os.access(deprecated_profile_file, os.R_OK): - return False - dcontent = io.open(_unicode_encode(deprecated_profile_file, + eprefix = settings["EPREFIX"] + for x in reversed(settings.profiles): + deprecated_profile_file = os.path.join(x, "deprecated") + if os.access(deprecated_profile_file, os.R_OK): + break + else: + deprecated_profile_file = None + + if deprecated_profile_file is None: + deprecated_profile_file = os.path.join(config_root or "/", + DEPRECATED_PROFILE_FILE) + if not os.access(deprecated_profile_file, os.R_OK): + deprecated_profile_file = os.path.join(config_root or "/", + 'etc', 'make.profile', 'deprecated') + if not os.access(deprecated_profile_file, os.R_OK): + return + + with io.open(_unicode_encode(deprecated_profile_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace').readlines() + mode='r', encoding=_encodings['content'], errors='replace') as f: + dcontent = f.readlines() writemsg(colorize("BAD", _("\n!!! Your current profile is " "deprecated and not supported anymore.")) + "\n", noiselevel=-1) writemsg(colorize("BAD", _("!!! Use eselect profile to update your " @@ -30,13 +47,37 @@ def deprecated_profile_check(settings=None): writemsg(colorize("BAD", _("!!! Please refer to the " "Gentoo Upgrading Guide.")) + "\n", noiselevel=-1) return True - newprofile = dcontent[0] + newprofile = dcontent[0].rstrip("\n") writemsg(colorize("BAD", _("!!! Please upgrade to the " - "following profile if possible:")) + "\n", noiselevel=-1) - writemsg(8*" " + colorize("GOOD", newprofile) + "\n", noiselevel=-1) + "following profile if possible:")) + "\n\n", noiselevel=-1) + writemsg(8*" " + colorize("GOOD", newprofile) + "\n\n", noiselevel=-1) if len(dcontent) > 1: writemsg(_("To upgrade do the following steps:\n"), noiselevel=-1) for myline in dcontent[1:]: writemsg(myline, noiselevel=-1) writemsg("\n\n", noiselevel=-1) + else: + writemsg(_("You may use the following command to upgrade:\n\n"), noiselevel=-1) + writemsg(8*" " + colorize("INFORM", 'eselect profile set ' + + newprofile) + "\n\n", noiselevel=-1) + + if settings is not None: + main_repo_loc = settings.repositories.mainRepoLocation() + new_profile_path = os.path.join(main_repo_loc, + "profiles", newprofile.rstrip("\n")) + + if os.path.isdir(new_profile_path): + new_config = portage.config(config_root=config_root, + config_profile_path=new_profile_path, + eprefix=eprefix) + + if not new_config.profiles: + writemsg("\n %s %s\n" % (colorize("WARN", "*"), + _("You must update portage before you " + "can migrate to the above profile.")), noiselevel=-1) + writemsg(" %s %s\n\n" % (colorize("WARN", "*"), + _("In order to update portage, " + "run 'emerge --oneshot portage'.")), + noiselevel=-1) + return True diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo Binary files differindex 2b9362b..df43aab 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py index 8705639..e207ba8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['digestcheck'] @@ -6,6 +6,7 @@ __all__ = ['digestcheck'] import warnings from portage import os, _encodings, _unicode_decode +from portage.checksum import _hash_filter from portage.exception import DigestException, FileNotFound from portage.localization import _ from portage.output import EOutput @@ -28,6 +29,9 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 pkgdir = mysettings["O"] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None if mf is None: mf = mysettings.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(pkgdir))) @@ -38,15 +42,16 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: if mf.fhashdict.get("EBUILD"): eout.ebegin(_("checking ebuild checksums ;-)")) - mf.checkTypeHashes("EBUILD") + mf.checkTypeHashes("EBUILD", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("AUX"): eout.ebegin(_("checking auxfile checksums ;-)")) - mf.checkTypeHashes("AUX") + mf.checkTypeHashes("AUX", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("MISC"): eout.ebegin(_("checking miscfile checksums ;-)")) - mf.checkTypeHashes("MISC", ignoreMissingFiles=True) + mf.checkTypeHashes("MISC", ignoreMissingFiles=True, + hash_filter=hash_filter) eout.eend(0) for f in myfiles: eout.ebegin(_("checking %s ;-)") % f) @@ -58,7 +63,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("\n!!! Missing digest for '%s'\n") % (f,), noiselevel=-1) return 0 - mf.checkFileHashes(ftype, f) + mf.checkFileHashes(ftype, f, hash_filter=hash_filter) eout.eend(0) except FileNotFound as e: eout.eend(1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo Binary files differindex 66987a2..c6a8d4e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py index 6ad3397..95d02db 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py @@ -112,67 +112,64 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): missing_files.append(myfile) continue - if missing_files: - for myfile in missing_files: - uris = set() - all_restrict = set() - for cpv in distfiles_map[myfile]: - uris.update(myportdb.getFetchMap( - cpv, mytree=mytree)[myfile]) - restrict = myportdb.aux_get(cpv, ['RESTRICT'], - mytree=mytree)[0] - # Here we ignore conditional parts of RESTRICT since - # they don't apply unconditionally. Assume such - # conditionals only apply on the client side where - # digestgen() does not need to be called. - all_restrict.update(use_reduce(restrict, - flat=True, matchnone=True)) - - # fetch() uses CATEGORY and PF to display a message - # when fetch restriction is triggered. - cat, pf = catsplit(cpv) - mysettings["CATEGORY"] = cat - mysettings["PF"] = pf - - # fetch() uses PORTAGE_RESTRICT to control fetch - # restriction, which is only applied to files that - # are not fetchable via a mirror:// URI. - mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) - - try: - st = os.stat(os.path.join( - mysettings["DISTDIR"],myfile)) - except OSError: - st = None - - if not fetch({myfile : uris}, mysettings): - myebuild = os.path.join(mysettings["O"], - catsplit(cpv)[1] + ".ebuild") - spawn_nofetch(myportdb, myebuild) - writemsg(_("!!! Fetch failed for %s, can't update " - "Manifest\n") % myfile, noiselevel=-1) - if myfile in dist_hashes and \ - st is not None and st.st_size > 0: - # stat result is obtained before calling fetch(), - # since fetch may rename the existing file if the - # digest does not match. - writemsg(_("!!! If you would like to " - "forcefully replace the existing " - "Manifest entry\n!!! for %s, use " - "the following command:\n") % myfile + \ - "!!! " + colorize("INFORM", - "ebuild --force %s manifest" % \ - os.path.basename(myebuild)) + "\n", - noiselevel=-1) - return 0 + for myfile in missing_files: + uris = set() + all_restrict = set() + for cpv in distfiles_map[myfile]: + uris.update(myportdb.getFetchMap( + cpv, mytree=mytree)[myfile]) + restrict = myportdb.aux_get(cpv, ['RESTRICT'], mytree=mytree)[0] + # Here we ignore conditional parts of RESTRICT since + # they don't apply unconditionally. Assume such + # conditionals only apply on the client side where + # digestgen() does not need to be called. + all_restrict.update(use_reduce(restrict, + flat=True, matchnone=True)) + + # fetch() uses CATEGORY and PF to display a message + # when fetch restriction is triggered. + cat, pf = catsplit(cpv) + mysettings["CATEGORY"] = cat + mysettings["PF"] = pf + + # fetch() uses PORTAGE_RESTRICT to control fetch + # restriction, which is only applied to files that + # are not fetchable via a mirror:// URI. + mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError: + st = None + + if not fetch({myfile : uris}, mysettings): + myebuild = os.path.join(mysettings["O"], + catsplit(cpv)[1] + ".ebuild") + spawn_nofetch(myportdb, myebuild) + writemsg(_("!!! Fetch failed for %s, can't update Manifest\n") + % myfile, noiselevel=-1) + if myfile in dist_hashes and \ + st is not None and st.st_size > 0: + # stat result is obtained before calling fetch(), + # since fetch may rename the existing file if the + # digest does not match. + cmd = colorize("INFORM", "ebuild --force %s manifest" % + os.path.basename(myebuild)) + writemsg((_( + "!!! If you would like to forcefully replace the existing Manifest entry\n" + "!!! for %s, use the following command:\n") % myfile) + + "!!! %s\n" % cmd, + noiselevel=-1) + return 0 + writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) try: mf.create(assumeDistHashesSometimes=True, assumeDistHashesAlways=( "assume-digests" in mysettings.features)) except FileNotFound as e: - writemsg(_("!!! File %s doesn't exist, can't update " - "Manifest\n") % e, noiselevel=-1) + writemsg(_("!!! File %s doesn't exist, can't update Manifest\n") + % e, noiselevel=-1) return 0 except PortagePackageException as e: writemsg(("!!! %s\n") % (e,), noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo Binary files differindex 66876ec..a4e9b62 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index 610172f..5351c52 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,14 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] +import grp import gzip import errno import io from itertools import chain import logging import os as _os +import platform +import pwd import re import signal import stat @@ -25,7 +30,13 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.digestcheck:digestcheck', 'portage.package.ebuild.digestgen:digestgen', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.dep._slot_operator:evaluate_slot_operator_equal_deps', 'portage.package.ebuild._spawn_nofetch:spawn_nofetch', + 'portage.util._desktop_entry:validate_desktop_entry', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) @@ -43,7 +54,7 @@ from portage.dep import Atom, check_required_use, \ from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ - eapi_has_pkg_pretend + eapi_has_pkg_pretend, _get_eapi_attrs from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ @@ -55,14 +66,13 @@ from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ writemsg, writemsg_stdout, write_atomic -from portage.util.lafilefixer import rewrite_lafile +from portage.util.lafilefixer import rewrite_lafile from portage.versions import _pkgsplit from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.EbuildSpawnProcess import EbuildSpawnProcess from _emerge.Package import Package -from _emerge.PollScheduler import PollScheduler from _emerge.RootConfig import RootConfig _unsandboxed_phases = frozenset([ @@ -72,6 +82,40 @@ _unsandboxed_phases = frozenset([ "prerm", "setup" ]) +# phases in which IPC with host is allowed +_ipc_phases = frozenset([ + "setup", "pretend", + "preinst", "postinst", "prerm", "postrm", +]) + +# phases in which networking access is allowed +_networked_phases = frozenset([ + # for VCS fetching + "unpack", + # + for network-bound IPC +] + list(_ipc_phases)) + +_phase_func_map = { + "config": "pkg_config", + "setup": "pkg_setup", + "nofetch": "pkg_nofetch", + "unpack": "src_unpack", + "prepare": "src_prepare", + "configure": "src_configure", + "compile": "src_compile", + "test": "src_test", + "install": "src_install", + "preinst": "pkg_preinst", + "postinst": "pkg_postinst", + "prerm": "pkg_prerm", + "postrm": "pkg_postrm", + "info": "pkg_info", + "pretend": "pkg_pretend", +} + +_vdb_use_conditional_keys = Package._dep_keys + \ + ('LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',) + def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): """ All proper ebuild phases which execute ebuild.sh are spawned @@ -81,8 +125,18 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): if phase in _unsandboxed_phases: kwargs['free'] = True + kwargs['ipc'] = 'ipc-sandbox' not in settings.features or \ + phase in _ipc_phases + kwargs['networked'] = 'network-sandbox' not in settings.features or \ + phase in _networked_phases + if phase == 'depend': kwargs['droppriv'] = 'userpriv' in settings.features + # It's not necessary to close_fds for this phase, since + # it should not spawn any daemons, and close_fds is + # best avoided since it can interact badly with some + # garbage collectors (see _setup_pipes docstring). + kwargs['close_fds'] = False if actionmap is not None and phase in actionmap: kwargs.update(actionmap[phase]["args"]) @@ -100,17 +154,24 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): settings['EBUILD_PHASE'] = phase try: - return spawn(cmd, settings, **kwargs) + return spawn(cmd, settings, **portage._native_kwargs(kwargs)) finally: settings.pop('EBUILD_PHASE', None) -def _spawn_phase(phase, settings, actionmap=None, **kwargs): - if kwargs.get('returnpid'): - return _doebuild_spawn(phase, settings, actionmap=actionmap, **kwargs) +def _spawn_phase(phase, settings, actionmap=None, returnpid=False, + logfile=None, **kwargs): + + if returnpid: + return _doebuild_spawn(phase, settings, actionmap=actionmap, + returnpid=returnpid, logfile=logfile, **kwargs) + # The logfile argument is unused here, since EbuildPhase uses + # the PORTAGE_LOG_FILE variable if set. ebuild_phase = EbuildPhase(actionmap=actionmap, background=False, - phase=phase, scheduler=PollScheduler().sched_iface, - settings=settings) + phase=phase, scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings, **kwargs) + ebuild_phase.start() ebuild_phase.wait() return ebuild_phase.returncode @@ -123,19 +184,28 @@ def _doebuild_path(settings, eapi=None): # Note: PORTAGE_BIN_PATH may differ from the global constant # when portage is reinstalling itself. portage_bin_path = settings["PORTAGE_BIN_PATH"] - eprefix = settings["EPREFIX"] + eprefix = portage.const.EPREFIX prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + overrides = [x for x in settings.get( + "__PORTAGE_TEST_PATH_OVERRIDE", "").split(":") if x] prefixes = [] if eprefix: prefixes.append(eprefix) prefixes.append("/") - path = [] + path = overrides + + if "xattr" in settings.features: + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "xattr")) - if eapi not in (None, "0", "1", "2", "3"): - path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + if eprefix and uid != 0 and "fakeroot" not in settings.features: + path.append(os.path.join(portage_bin_path, + "ebuild-helpers", "unprivileged")) + + if settings.get("USERLAND", "GNU") != "GNU": + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "bsd")) path.append(os.path.join(portage_bin_path, "ebuild-helpers")) path.extend(prerootpath) @@ -254,10 +324,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, if hasattr(mydbapi, 'repositories'): repo = mydbapi.repositories.get_repo_for_location(mytree) mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] - mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings['PORTAGE_ECLASS_LOCATIONS'] = repo.eclass_db.eclass_locations_string mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) + mysettings.pop("PORTDIR_OVERLAY", None) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) @@ -414,14 +485,14 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, - fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, +def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, listonly=0, + fetchonly=0, cleanup=0, dbkey=DeprecationWarning, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): """ Wrapper function that invokes specific ebuild phases through the spawning of ebuild.sh - + @param myebuild: name of the ebuild to invoke the phase on (CPV) @type myebuild: String @param mydo: Phase to run @@ -464,13 +535,13 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, @return: 1. 0 for success 2. 1 for error - + Most errors have an accompanying error message. - + listonly and fetchonly are only really necessary for operations involving 'fetch' prev_mtimes are only necessary for merge operations. Other variables may not be strictly required, many have defaults that are set inside of doebuild. - + """ if settings is None: @@ -478,17 +549,22 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings = settings myroot = settings['EROOT'] - if _unused is not None and _unused != mysettings['EROOT']: + if _unused is not DeprecationWarning: warnings.warn("The third parameter of the " - "portage.doebuild() is now unused. Use " - "settings['ROOT'] instead.", + "portage.doebuild() is deprecated. Instead " + "settings['EROOT'] is used.", + DeprecationWarning, stacklevel=2) + + if dbkey is not DeprecationWarning: + warnings.warn("portage.doebuild() called " + "with deprecated dbkey argument.", DeprecationWarning, stacklevel=2) if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" - - # chunked out deps for each phase, so that ebuild binary can use it + + # chunked out deps for each phase, so that ebuild binary can use it # to collapse targets down. actionmap_deps={ "pretend" : [], @@ -503,7 +579,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "package":["install"], "merge" :["install"], } - + if mydbapi is None: mydbapi = portage.db[myroot][tree].dbapi @@ -518,7 +594,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "fetch", "fetchall", "digest", "unpack", "prepare", "configure", "compile", "test", "install", "rpm", "qmerge", "merge", - "package","unmerge", "manifest"] + "package", "unmerge", "manifest", "nofetch"] if mydo not in validcommands: validcommands.sort() @@ -532,8 +608,11 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return 1 if returnpid and mydo != 'depend': - warnings.warn("portage.doebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + # This case is not supported, since it bypasses the EbuildPhase class + # which implements important functionality (including post phase hooks + # and IPC for things like best/has_version and die). + warnings.warn("portage.doebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -541,9 +620,6 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchall = 1 mydo = "fetch" - parallel_fetchonly = mydo in ("fetch", "fetchall") and \ - "PORTAGE_PARALLEL_FETCHONLY" in mysettings - if mydo not in clean_phases and not os.path.exists(myebuild): writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1) @@ -650,7 +726,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # we can temporarily override PORTAGE_TMPDIR with a random temp dir # so that there's no need for locking and it can be used even if the # user isn't in the portage group. - if mydo in ("info",): + if not returnpid and mydo in ("info",): tmpdir = tempfile.mkdtemp() tmpdir_orig = mysettings["PORTAGE_TMPDIR"] mysettings["PORTAGE_TMPDIR"] = tmpdir @@ -661,9 +737,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in clean_phases: builddir_lock = None if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -679,42 +756,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if returnpid: return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - elif isinstance(dbkey, dict): - warnings.warn("portage.doebuild() called " + \ - "with dict dbkey argument. This usage will " + \ - "not be supported in the future.", - DeprecationWarning, stacklevel=2) - mysettings["dbkey"] = "" - pr, pw = os.pipe() - fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), - 9:pw} - mypids = _spawn_phase(mydo, mysettings, returnpid=True, - fd_pipes=fd_pipes) - os.close(pw) # belongs exclusively to the child process now - f = os.fdopen(pr, 'rb', 0) - for k, v in zip(auxdbkeys, - (_unicode_decode(line).rstrip('\n') for line in f)): - dbkey[k] = v - f.close() - retval = os.waitpid(mypids[0], 0)[1] - portage.process.spawned_pids.remove(mypids[0]) - # If it got a signal, return the signal that was sent, but - # shift in order to distinguish it from a return value. (just - # like portage.process.spawn() would do). - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - # Otherwise, return its exit code. - retval = retval >> 8 - if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): - # Don't trust bash's returncode if the - # number of lines is incorrect. - retval = 1 - return retval - elif dbkey: + elif dbkey and dbkey is not DeprecationWarning: mysettings["dbkey"] = dbkey else: mysettings["dbkey"] = \ @@ -723,14 +765,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - # Validate dependency metadata here to ensure that ebuilds with invalid - # data are never installed via the ebuild command. Don't bother when - # returnpid == True since there's no need to do this every time emerge - # executes a phase. + elif mydo == "nofetch": + + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + return spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) + if tree == "porttree": - rval = _validate_deps(mysettings, myroot, mydo, mydbapi) - if rval != os.EX_OK: - return rval + + if not returnpid: + # Validate dependency metadata here to ensure that ebuilds with + # invalid data are never installed via the ebuild command. Skip + # this when returnpid is True (assume the caller handled it). + rval = _validate_deps(mysettings, myroot, mydo, mydbapi) + if rval != os.EX_OK: + return rval else: # FEATURES=noauto only makes sense for porttree, and we don't want @@ -739,20 +792,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if "noauto" in mysettings.features: mysettings.features.discard("noauto") - # The info phase is special because it uses mkdtemp so and - # user (not necessarily in the portage group) can run it. - if mydo not in ('info',) and \ + # If we are not using a private temp dir, then check access + # to the global temp dir. + if tmpdir is None and \ mydo not in _doebuild_commands_without_builddir: rval = _check_temp_dir(mysettings) if rval != os.EX_OK: return rval if mydo == "unmerge": + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) phases_to_run = set() - if "noauto" in mysettings.features or \ + if returnpid or \ + "noauto" in mysettings.features or \ mydo not in actionmap_deps: phases_to_run.add(mydo) else: @@ -803,9 +861,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if newstuff: if builddir_lock is None and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -823,12 +882,12 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # in order to satisfy the sane $PWD requirement (from bug #239560) # when pkg_nofetch is spawned. have_build_dirs = False - if not parallel_fetchonly and \ - mydo not in ('digest', 'fetch', 'help', 'manifest'): + if mydo not in ('digest', 'fetch', 'help', 'manifest'): if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() mystatus = prepare_build_dirs(myroot, mysettings, cleanup) @@ -871,9 +930,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, else: vardb = vartree.dbapi cpv = mysettings.mycpv - cp = portage.versions.cpv_getkey(cpv) - slot = mysettings["SLOT"] - cpv_slot = cp + ":" + slot + cpv_slot = "%s%s%s" % \ + (cpv.cp, portage.dep._slot_separator, cpv.slot) mysettings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(match) \ for match in vardb.match(cpv_slot) + \ @@ -883,8 +941,16 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # the sandbox -- and stop now. if mydo in ("config", "help", "info", "postinst", "preinst", "pretend", "postrm", "prerm"): - return _spawn_phase(mydo, mysettings, - fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + if mydo in ("preinst", "postinst"): + env_file = os.path.join(os.path.dirname(mysettings["EBUILD"]), + "environment.bz2") + if os.path.isfile(env_file): + mysettings["PORTAGE_UPDATE_ENV"] = env_file + try: + return _spawn_phase(mydo, mysettings, + fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + finally: + mysettings.pop("PORTAGE_UPDATE_ENV", None) mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) @@ -925,7 +991,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=True, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings) + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) if listonly: # The convention for listonly mode is to report # success in any case, even though fetch() may @@ -957,11 +1024,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mf = None _doebuild_manifest_cache = None return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo != 'fetch' and \ - "digest" in mysettings.features: - # Don't do this when called by emerge or when called just - # for fetch (especially parallel-fetch) since it's not needed - # and it can interfere with parallel tasks. + elif "digest" in mysettings.features: mf = None _doebuild_manifest_cache = None digestgen(mysettings=mysettings, myportdb=mydbapi) @@ -970,14 +1033,17 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in ("digest", "manifest"): return 1 + if mydo == "fetch": + # Return after digestgen for FEATURES=digest support. + # Return before digestcheck, since fetch() already + # checked any relevant digests. + return 0 + # See above comment about fetching only when needed if tree == 'porttree' and \ not digestcheck(checkme, mysettings, "strict" in features, mf=mf): return 1 - if mydo == "fetch": - return 0 - # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI if tree == 'porttree' and \ ((mydo != "setup" and "noauto" not in features) \ @@ -993,7 +1059,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if len(actionmap_deps.get(x, [])): actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) - if mydo in actionmap: + regular_actionmap_phase = mydo in actionmap + + if regular_actionmap_phase: bintree = None if mydo == "package": # Make sure the package directory exists before executing @@ -1017,6 +1085,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, actionmap, mysettings, debug, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) + if returnpid and isinstance(retval, list): + return retval + if retval == os.EX_OK: if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, @@ -1028,7 +1099,15 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, except OSError: pass - elif mydo=="qmerge": + elif returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + if regular_actionmap_phase: + # handled above + pass + elif mydo == "qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild if not os.path.exists( @@ -1045,7 +1124,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, - mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) + mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) elif mydo=="merge": retval = spawnebuild("install", actionmap, mysettings, debug, alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, @@ -1061,7 +1141,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, - vartree=vartree, prev_mtimes=prev_mtimes) + vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) + else: writemsg_stdout(_("!!! Unknown mydo: %s\n") % mydo, noiselevel=-1) return 1 @@ -1161,7 +1243,9 @@ def _prepare_env_file(settings): """ env_extractor = BinpkgEnvExtractor(background=False, - scheduler=PollScheduler().sched_iface, settings=settings) + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings) if env_extractor.dest_env_exists(): # There are lots of possible states when doebuild() @@ -1243,7 +1327,7 @@ def _spawn_actionmap(settings): misc_sh_binary = os.path.join(portage_bin_path, os.path.basename(MISC_SH_BINARY)) ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" - misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" + misc_sh = _shell_quote(misc_sh_binary) + " __dyn_%s" # args are for the to spawn function actionmap = { @@ -1299,10 +1383,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): if not pkg.built and \ mydo not in ("digest", "help", "manifest") and \ - pkg.metadata["REQUIRED_USE"] and \ - eapi_has_required_use(pkg.metadata["EAPI"]): - result = check_required_use(pkg.metadata["REQUIRED_USE"], - pkg.use.enabled, pkg.iuse.is_valid_flag) + pkg._metadata["REQUIRED_USE"] and \ + eapi_has_required_use(pkg.eapi): + result = check_required_use(pkg._metadata["REQUIRED_USE"], + pkg.use.enabled, pkg.iuse.is_valid_flag, eapi=pkg.eapi) if not result: reduced_noise = result.tounicode() writemsg("\n %s\n" % _("The following REQUIRED_USE flag" + \ @@ -1310,7 +1394,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): writemsg(" %s\n" % reduced_noise, noiselevel=-1) normalized_required_use = \ - " ".join(pkg.metadata["REQUIRED_USE"].split()) + " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg("\n %s\n" % _("The above constraints " + \ "are a subset of the following complete expression:"), @@ -1325,7 +1409,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): # XXX This would be to replace getstatusoutput completely. # XXX Issue: cannot block execution. Deadlock condition. -def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords): +def spawn(mystring, mysettings, debug=False, free=False, droppriv=False, + sesandbox=False, fakeroot=False, networked=True, ipc=True, **keywords): """ Spawn a subprocess with extra portage-specific options. Optiosn include: @@ -1355,6 +1440,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @type sesandbox: Boolean @param fakeroot: Run this command with faked root privileges @type fakeroot: Boolean + @param networked: Run this command with networking access enabled + @type networked: Boolean + @param ipc: Run this command with host IPC access enabled + @type ipc: Boolean @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer @@ -1367,29 +1456,90 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero fd_pipes = keywords.get("fd_pipes") if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # In some cases the above print statements don't flush stdout, so # it needs to be flushed before allowing a child process to use it # so that output always shows in the correct order. - stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno()) + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: - sys.stdout.flush() - sys.stderr.flush() + sys.__stdout__.flush() + sys.__stderr__.flush() break features = mysettings.features + + # Use Linux namespaces if available + if uid == 0 and platform.system() == 'Linux': + keywords['unshare_net'] = not networked + keywords['unshare_ipc'] = not ipc + # TODO: Enable fakeroot to be used together with droppriv. The # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and uid == 0 and portage_gid and portage_uid and \ - hasattr(os, "setgroups"): - keywords.update({"uid":portage_uid,"gid":portage_gid, - "groups":userpriv_groups,"umask":0o02}) + portage_build_uid = os.getuid() + portage_build_gid = os.getgid() + if uid == 0 and portage_uid and portage_gid and hasattr(os, "setgroups"): + if droppriv: + keywords.update({ + "uid": portage_uid, + "gid": portage_gid, + "groups": userpriv_groups, + "umask": 0o02 + }) + + # Adjust pty ownership so that subprocesses + # can directly access /dev/fd/{1,2}. + stdout_fd = fd_pipes.get(1) + if stdout_fd is not None: + try: + subprocess_tty = _os.ttyname(stdout_fd) + except OSError: + pass + else: + try: + parent_tty = _os.ttyname(sys.__stdout__.fileno()) + except OSError: + parent_tty = None + + if subprocess_tty != parent_tty: + _os.chown(subprocess_tty, + int(portage_uid), int(portage_gid)) + + if "userpriv" in features and "userpriv" not in mysettings["PORTAGE_RESTRICT"].split() and secpass >= 2: + # Since Python 3.4, getpwuid and getgrgid + # require int type (no proxies). + portage_build_uid = int(portage_uid) + portage_build_gid = int(portage_gid) + + if "PORTAGE_BUILD_USER" not in mysettings: + user = None + try: + user = pwd.getpwuid(portage_build_uid).pw_name + except KeyError: + if portage_build_uid == 0: + user = "root" + elif portage_build_uid == portage_uid: + user = portage.data._portage_username + if user is not None: + mysettings["PORTAGE_BUILD_USER"] = user + + if "PORTAGE_BUILD_GROUP" not in mysettings: + group = None + try: + group = grp.getgrgid(portage_build_gid).gr_name + except KeyError: + if portage_build_gid == 0: + group = "root" + elif portage_build_gid == portage_gid: + group = portage.data._portage_grpname + if group is not None: + mysettings["PORTAGE_BUILD_GROUP"] = group + if not free: free=((droppriv and "usersandbox" not in features) or \ (not droppriv and "sandbox" not in features and \ @@ -1423,12 +1573,15 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero mysettings["PORTAGE_SANDBOX_T"]) if keywords.get("returnpid"): - return spawn_func(mystring, env=mysettings.environ(), **keywords) + return spawn_func(mystring, env=mysettings.environ(), + **portage._native_kwargs(keywords)) proc = EbuildSpawnProcess( background=False, args=mystring, - scheduler=PollScheduler().sched_iface, spawn_func=spawn_func, - settings=mysettings, **keywords) + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + spawn_func=spawn_func, + settings=mysettings, **portage._native_kwargs(keywords)) proc.start() proc.wait() @@ -1440,8 +1593,8 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, logfile=None, fd_pipes=None, returnpid=False): if returnpid: - warnings.warn("portage.spawnebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + warnings.warn("portage.spawnebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -1534,7 +1687,52 @@ def _check_build_log(mysettings, out=None): configure_opts_warn = [] configure_opts_warn_re = re.compile( - r'^configure: WARNING: [Uu]nrecognized options: ') + r'^configure: WARNING: [Uu]nrecognized options: (.*)') + + qa_configure_opts = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_CONFIGURE_OPTIONS"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_configure_opts_f: + qa_configure_opts = qa_configure_opts_f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_configure_opts = qa_configure_opts.split() + if qa_configure_opts: + if len(qa_configure_opts) > 1: + qa_configure_opts = "|".join("(%s)" % x for x in qa_configure_opts) + qa_configure_opts = "^(%s)$" % qa_configure_opts + else: + qa_configure_opts = "^%s$" % qa_configure_opts[0] + qa_configure_opts = re.compile(qa_configure_opts) + + qa_am_maintainer_mode = [] + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_AM_MAINTAINER_MODE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_am_maintainer_mode_f: + qa_am_maintainer_mode = [x for x in + qa_am_maintainer_mode_f.read().splitlines() if x] + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + if qa_am_maintainer_mode: + if len(qa_am_maintainer_mode) > 1: + qa_am_maintainer_mode = \ + "|".join("(%s)" % x for x in qa_am_maintainer_mode) + qa_am_maintainer_mode = "^(%s)$" % qa_am_maintainer_mode + else: + qa_am_maintainer_mode = "^%s$" % qa_am_maintainer_mode[0] + qa_am_maintainer_mode = re.compile(qa_am_maintainer_mode) # Exclude output from dev-libs/yaz-3.0.47 which looks like this: # @@ -1556,7 +1754,9 @@ def _check_build_log(mysettings, out=None): for line in f: line = _unicode_decode(line) if am_maintainer_mode_re.search(line) is not None and \ - am_maintainer_mode_exclude_re.search(line) is None: + am_maintainer_mode_exclude_re.search(line) is None and \ + (not qa_am_maintainer_mode or + qa_am_maintainer_mode.search(line) is None): am_maintainer_mode.append(line.rstrip("\n")) if bash_command_not_found_re.match(line) is not None and \ @@ -1566,8 +1766,11 @@ def _check_build_log(mysettings, out=None): if helper_missing_file_re.match(line) is not None: helper_missing_file.append(line.rstrip("\n")) - if configure_opts_warn_re.match(line) is not None: - configure_opts_warn.append(line.rstrip("\n")) + m = configure_opts_warn_re.match(line) + if m is not None: + for x in m.group(1).split(", "): + if not qa_configure_opts or qa_configure_opts.match(x) is None: + configure_opts_warn.append(x) if make_jobserver_re.match(line) is not None: make_jobserver.append(line.rstrip("\n")) @@ -1616,7 +1819,7 @@ def _check_build_log(mysettings, out=None): if configure_opts_warn: msg = [_("QA Notice: Unrecognized configure options:")] msg.append("") - msg.extend("\t" + line for line in configure_opts_warn) + msg.extend("\t%s" % x for x in configure_opts_warn) _eqawarn(msg) if make_jobserver: @@ -1629,7 +1832,7 @@ def _check_build_log(mysettings, out=None): if f_real is not None: f_real.close() -def _post_src_install_chost_fix(settings): +def _post_src_install_write_metadata(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial @@ -1637,10 +1840,16 @@ def _post_src_install_chost_fix(settings): due to local environment settings like in bug #386829. """ + eapi_attrs = _get_eapi_attrs(settings.configdict['pkg']['EAPI']) + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') - for k in ('IUSE',): - v = settings.get(k) + metadata_keys = ['IUSE'] + if eapi_attrs.iuse_effective: + metadata_keys.append('IUSE_EFFECTIVE') + + for k in metadata_keys: + v = settings.configdict['pkg'].get(k) if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') @@ -1652,9 +1861,59 @@ def _post_src_install_chost_fix(settings): if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') -_vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', - 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) -_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) + with io.open(_unicode_encode(os.path.join(build_info_dir, + 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write("%.0f\n" % (time.time(),)) + + use = frozenset(settings['PORTAGE_USE'].split()) + for k in _vdb_use_conditional_keys: + v = settings.configdict['pkg'].get(k) + filename = os.path.join(build_info_dir, k) + if v is None: + try: + os.unlink(filename) + except OSError: + pass + continue + + if k.endswith('DEPEND'): + if eapi_attrs.slot_operator: + continue + token_class = Atom + else: + token_class = None + + v = use_reduce(v, uselist=use, token_class=token_class) + v = paren_enclose(v) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) + + if eapi_attrs.slot_operator: + deps = evaluate_slot_operator_equal_deps(settings, use, QueryCommand.get_db()) + for k, v in deps.items(): + filename = os.path.join(build_info_dir, k) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) def _preinst_bsdflags(mysettings): if bsd_chflags: @@ -1696,6 +1955,33 @@ def _post_src_install_uid_fix(mysettings, out): destdir = mysettings["D"] ed_len = len(mysettings["ED"]) unicode_errors = [] + desktop_file_validate = \ + portage.process.find_binary("desktop-file-validate") is not None + xdg_dirs = mysettings.get('XDG_DATA_DIRS', '/usr/share').split(':') + xdg_dirs = tuple(os.path.join(i, "applications") + os.sep + for i in xdg_dirs if i) + + qa_desktop_file = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_DESKTOP_FILE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + qa_desktop_file = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_desktop_file = qa_desktop_file.split() + if qa_desktop_file: + if len(qa_desktop_file) > 1: + qa_desktop_file = "|".join("(%s)" % x for x in qa_desktop_file) + qa_desktop_file = "^(%s)$" % qa_desktop_file + else: + qa_desktop_file = "^%s$" % qa_desktop_file[0] + qa_desktop_file = re.compile(qa_desktop_file) while True: @@ -1704,6 +1990,7 @@ def _post_src_install_uid_fix(mysettings, out): counted_inodes = set() fixlafiles_announced = False fixlafiles = "fixlafiles" in mysettings.features + desktopfile_errors = [] for parent, dirs, files in os.walk(destdir): try: @@ -1743,6 +2030,16 @@ def _post_src_install_uid_fix(mysettings, out): else: fpath = os.path.join(parent, fname) + fpath_relative = fpath[ed_len - 1:] + if desktop_file_validate and fname.endswith(".desktop") and \ + os.path.isfile(fpath) and \ + fpath_relative.startswith(xdg_dirs) and \ + not (qa_desktop_file and qa_desktop_file.match(fpath_relative.strip(os.sep)) is not None): + + desktop_validate = validate_desktop_entry(fpath) + if desktop_validate: + desktopfile_errors.extend(desktop_validate) + if fixlafiles and \ fname.endswith(".la") and os.path.isfile(fpath): f = open(_unicode_encode(fpath, @@ -1809,6 +2106,11 @@ def _post_src_install_uid_fix(mysettings, out): if not unicode_error: break + if desktopfile_errors: + for l in _merge_desktopfile_error(desktopfile_errors): + l = l.replace(mysettings["ED"], '/') + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) + if unicode_errors: for l in _merge_unicode_error(unicode_errors): eqawarn(l, phase='install', key=mysettings.mycpv, out=out) @@ -1820,47 +2122,9 @@ def _post_src_install_uid_fix(mysettings, out): 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') - f.write(_unicode_decode(str(size) + '\n')) + f.write('%d\n' % size) f.close() - f = io.open(_unicode_encode(os.path.join(build_info_dir, - 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode("%.0f\n" % (time.time(),))) - f.close() - - use = frozenset(mysettings['PORTAGE_USE'].split()) - for k in _vdb_use_conditional_keys: - v = mysettings.configdict['pkg'].get(k) - filename = os.path.join(build_info_dir, k) - if v is None: - try: - os.unlink(filename) - except OSError: - pass - continue - - if k.endswith('DEPEND'): - token_class = Atom - else: - token_class = None - - v = use_reduce(v, uselist=use, token_class=token_class) - v = paren_enclose(v) - if not v: - try: - os.unlink(filename) - except OSError: - pass - continue - f = io.open(_unicode_encode(os.path.join(build_info_dir, - k), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode(v + '\n')) - f.close() - _reapply_bsdflags_to_image(mysettings) def _reapply_bsdflags_to_image(mysettings): @@ -2009,6 +2273,20 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) +def _merge_desktopfile_error(errors): + lines = [] + + msg = _("QA Notice: This package installs one or more .desktop files " + "that do not pass validation.") + lines.extend(wrap(msg, 72)) + + lines.append("") + errors.sort() + lines.extend("\t" + x for x in errors) + lines.append("") + + return lines + def _merge_unicode_error(errors): lines = [] @@ -2065,11 +2343,6 @@ def _handle_self_update(settings, vardb): if settings["ROOT"] == "/" and \ portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): - inherited = frozenset(settings.get('INHERITED', '').split()) - if not vardb.cpv_exists(cpv) or \ - '9999' in cpv or \ - 'git' in inherited or \ - 'git-2' in inherited: - _prepare_self_update(settings) - return True + _prepare_self_update(settings) + return True return False diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo Binary files differindex a6ebb1d..846d99a 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.py b/portage_with_autodep/pym/portage/package/ebuild/fetch.py index b795b28..5316f03 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -14,6 +14,10 @@ import stat import sys import tempfile +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -25,7 +29,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode -from portage.checksum import hashfunc_map, perform_md5, verify_all +from portage.checksum import (hashfunc_map, perform_md5, verify_all, + _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter) from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -63,9 +68,9 @@ def _spawn_fetch(settings, args, **kwargs): if "fd_pipes" not in kwargs: kwargs["fd_pipes"] = { - 0 : sys.stdin.fileno(), - 1 : sys.stdout.fileno(), - 2 : sys.stdout.fileno(), + 0 : portage._get_stdin().fileno(), + 1 : sys.__stdout__.fileno(), + 2 : sys.__stdout__.fileno(), } if "userfetch" in settings.features and \ @@ -184,7 +189,7 @@ def _check_digests(filename, digests, show_errors=1): return False return True -def _check_distfile(filename, digests, eout, show_errors=1): +def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None): """ @return a tuple of (match, stat_obj) where match is True if filename matches all given digests (if any) and stat_obj is a stat result, or @@ -210,6 +215,9 @@ def _check_distfile(filename, digests, eout, show_errors=1): # Zero-byte distfiles are always invalid. return (False, st) else: + digests = _filter_unaccelarated_hashes(digests) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) if _check_digests(filename, digests, show_errors=show_errors): eout.ebegin("%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests)))) @@ -339,7 +347,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, _("!!! For fetching to a read-only filesystem, " "locking should be turned off.\n")), noiselevel=-1) writemsg(_("!!! This can be done by adding -distlocks to " - "FEATURES in /etc/make.conf\n"), noiselevel=-1) + "FEATURES in /etc/portage/make.conf\n"), noiselevel=-1) # use_locks = 0 # local mirrors are always added @@ -353,6 +361,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if try_mirrors: mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1" if skip_manifest: allow_missing_digests = True @@ -395,12 +406,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for myfile, uri_set in myuris.items(): for myuri in uri_set: file_uri_tuples.append((myfile, myuri)) + if not uri_set: + file_uri_tuples.append((myfile, None)) else: for myuri in myuris: - file_uri_tuples.append((os.path.basename(myuri), myuri)) + if urlparse(myuri).scheme: + file_uri_tuples.append((os.path.basename(myuri), myuri)) + else: + file_uri_tuples.append((os.path.basename(myuri), None)) filedict = OrderedDict() - primaryuri_indexes={} primaryuri_dict = {} thirdpartymirror_uris = {} for myfile, myuri in file_uri_tuples: @@ -408,6 +423,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, filedict[myfile]=[] for y in range(0,len(locations)): filedict[myfile].append(locations[y]+"/distfiles/"+myfile) + if myuri is None: + continue if myuri[:9]=="mirror://": eidx = myuri.find("/", 9) if eidx != -1: @@ -422,10 +439,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # now try the official mirrors if mirrorname in thirdpartymirrors: - random.shuffle(thirdpartymirrors[mirrorname]) - uris = [locmirr.rstrip("/") + "/" + path \ for locmirr in thirdpartymirrors[mirrorname]] + random.shuffle(uris) filedict[myfile].extend(uris) thirdpartymirror_uris.setdefault(myfile, []).extend(uris) @@ -438,26 +454,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if restrict_fetch or force_mirror: # Only fetch from specific mirrors is allowed. continue - if "primaryuri" in restrict: - # Use the source site first. - if myfile in primaryuri_indexes: - primaryuri_indexes[myfile] += 1 - else: - primaryuri_indexes[myfile] = 0 - filedict[myfile].insert(primaryuri_indexes[myfile], myuri) - else: - filedict[myfile].append(myuri) primaryuris = primaryuri_dict.get(myfile) if primaryuris is None: primaryuris = [] primaryuri_dict[myfile] = primaryuris primaryuris.append(myuri) + # Order primaryuri_dict values to match that in SRC_URI. + for uris in primaryuri_dict.values(): + uris.reverse() + # Prefer thirdpartymirrors over normal mirrors in cases when # the file does not yet exist on the normal mirrors. for myfile, uris in thirdpartymirror_uris.items(): primaryuri_dict.setdefault(myfile, []).extend(uris) + # Now merge primaryuri values into filedict (includes mirrors + # explicitly referenced in SRC_URI). + if "primaryuri" in restrict: + for myfile, uris in filedict.items(): + filedict[myfile] = primaryuri_dict.get(myfile, []) + uris + else: + for myfile in filedict: + filedict[myfile] += primaryuri_dict.get(myfile, []) + can_fetch=True if listonly: @@ -635,7 +655,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" match, mystat = _check_distfile( - myfile_path, pruned_digests, eout) + myfile_path, pruned_digests, eout, hash_filter=hash_filter) if match: # Skip permission adjustment for symlinks, since we don't # want to modify anything outside of the primary DISTDIR, @@ -707,7 +727,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for x in ro_distdirs: filename = os.path.join(x, myfile) match, mystat = _check_distfile( - filename, pruned_digests, eout) + filename, pruned_digests, eout, hash_filter=hash_filter) if match: readonly_file = filename break @@ -732,7 +752,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, "remaining space.\n"), noiselevel=-1) if userfetch: writemsg(_("!!! You may set FEATURES=\"-userfetch\"" - " in /etc/make.conf in order to fetch with\n" + " in /etc/portage/make.conf in order to fetch with\n" "!!! superuser privileges.\n"), noiselevel=-1) if fsmirrors and not os.path.exists(myfile_path) and has_space: @@ -793,8 +813,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout.eend(0) continue else: - verified_ok, reason = verify_all( - myfile_path, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Previously fetched" " file: '%s'\n") % myfile, noiselevel=-1) @@ -816,7 +838,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = \ mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: digests = list(digests) digests.sort() @@ -844,8 +865,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, protocol = loc[0:loc.find("://")] global_config_path = GLOBAL_CONFIG_PATH - if mysettings['EPREFIX']: - global_config_path = os.path.join(mysettings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) missing_file_param = False @@ -954,11 +975,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, writemsg_stdout(_(">>> Downloading '%s'\n") % \ _hide_url_passwd(loc)) variables = { - "DISTDIR": mysettings["DISTDIR"], "URI": loc, "FILE": myfile } + for k in ("DISTDIR", "PORTAGE_SSH_OPTS"): + try: + variables[k] = mysettings[k] + except KeyError: + pass + myfetch = shlex_split(locfetch) myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 @@ -1051,7 +1077,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... - verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) @@ -1085,7 +1114,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, else: eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: eout.ebegin("%s %s ;-)" % \ (myfile, " ".join(sorted(digests)))) diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo Binary files differindex 3bd81df..6e2c242 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py index 8a88c2f..70a6bf2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingreason'] @@ -6,13 +6,12 @@ __all__ = ['getmaskingreason'] import portage from portage import os from portage.const import USER_CONFIG_PATH -from portage.dep import Atom, match_from_list, _slot_separator, _repo_separator +from portage.dep import Atom, match_from_list from portage.exception import InvalidAtom from portage.localization import _ from portage.repository.config import _gen_valid_repo from portage.util import grablines, normalize_path -from portage.versions import catpkgsplit -from _emerge.Package import Package +from portage.versions import catpkgsplit, _pkg_str def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): @@ -60,23 +59,20 @@ def getmaskingreason(mycpv, metadata=None, settings=None, # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv - if metadata is not None: - pkg = "".join((mycpv, _slot_separator, metadata["SLOT"])) - # At this point myrepo should be None, a valid name, or - # Package.UNKNOWN_REPO which we ignore. - if myrepo is not None and myrepo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, myrepo)) + try: + pkg.slot + except AttributeError: + pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo) + cpv_slot_list = [pkg] - mycp=mysplit[0]+"/"+mysplit[1] + mycp = pkg.cp - # XXX- This is a temporary duplicate of code from the config constructor. - locations = [os.path.join(settings["PORTDIR"], "profiles")] + locations = [] + if pkg.repo in settings.repositories: + for repo in settings.repositories[pkg.repo].masters + (settings.repositories[pkg.repo],): + locations.append(os.path.join(repo.location, "profiles")) locations.extend(settings.profiles) - for ov in settings["PORTDIR_OVERLAY"].split(): - profdir = os.path.join(normalize_path(ov), "profiles") - if os.path.isdir(profdir): - locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo Binary files differindex 1614244..6c0073d 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py index 9bf605d..c8954aa 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,12 +1,15 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['getmaskingstatus'] import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated +from portage.exception import InvalidDependString from portage.localization import _ from portage.package.ebuild.config import config from portage.versions import catpkgsplit, _pkg_str @@ -48,7 +51,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): # emerge passed in a Package instance pkg = mycpv mycpv = pkg.cpv - metadata = pkg.metadata + metadata = pkg._metadata installed = pkg.installed if metadata is None: @@ -65,10 +68,11 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" - if not hasattr(mycpv, 'slot'): + try: + mycpv.slot + except AttributeError: try: - mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], - repo=metadata.get('repository')) + mycpv = _pkg_str(mycpv, metadata=metadata, settings=settings) except portage.exception.InvalidData: raise ValueError(_("invalid CPV: %s") % mycpv) @@ -83,6 +87,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] + restrict = metadata["RESTRICT"] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: @@ -122,6 +127,13 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): if gp=="*": kmask=None break + elif gp == "~*": + for x in pgroups: + if x[:1] == "~": + kmask = None + break + if kmask is None: + break elif gp=="-"+myarch and myarch in pgroups: kmask="-"+myarch break @@ -161,6 +173,15 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): except portage.exception.InvalidDependString as e: rValue.append(_MaskReason("invalid", "PROPERTIES: "+str(e))) + try: + missing_restricts = settings._getMissingRestrict(mycpv, metadata) + if missing_restricts: + msg = list(missing_restricts) + msg.append("in RESTRICT") + rValue.append(_MaskReason("RESTRICT", " ".join(msg))) + except InvalidDependString as e: + rValue.append(_MaskReason("invalid", "RESTRICT: %s" % (e,))) + # Only show KEYWORDS masks for installed packages # if they're not masked for any other reason. if kmask and (not installed or not rValue): diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo Binary files differindex 9cf1d9d..27bb3c7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py index b8fbdc5..6782160 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['prepare_build_dirs'] import errno @@ -338,12 +340,12 @@ def _prepare_workdir(mysettings): try: _ensure_log_subdirs(logdir, log_subdir) except PortageException as e: - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) if os.access(log_subdir, os.W_OK): logdir_subdir_ok = True else: - writemsg(_unicode_decode("!!! %s: %s\n") % + writemsg("!!! %s: %s\n" % (_("Permission Denied"), log_subdir), noiselevel=-1) tmpdir_log_path = os.path.join( diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo Binary files differindex 2dcfaea..fd6d446 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo diff --git a/portage_with_autodep/pym/portage/process.py b/portage_with_autodep/pym/portage/process.py index d7d1037..7334c00 100644 --- a/portage_with_autodep/pym/portage/process.py +++ b/portage_with_autodep/pym/portage/process.py @@ -1,25 +1,30 @@ # portage.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import atexit import errno +import fcntl import platform import signal +import socket +import struct import sys import traceback +import os as _os from portage import os from portage import _encodings from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:dump_traceback', + 'portage.util:dump_traceback,writemsg', ) from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY from portage.exception import CommandNotFound +from portage.util._ctypes import find_library, LoadLibrary, ctypes try: import resource @@ -30,10 +35,32 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str -if os.path.isdir("/proc/%i/fd" % os.getpid()): +# Support PEP 446 for Python >=3.4 +try: + _set_inheritable = _os.set_inheritable +except AttributeError: + _set_inheritable = None + +try: + _FD_CLOEXEC = fcntl.FD_CLOEXEC +except AttributeError: + _FD_CLOEXEC = None + +# Prefer /proc/self/fd if available (/dev/fd +# doesn't work on solaris, see bug #474536). +for _fd_dir in ("/proc/self/fd", "/dev/fd"): + if os.path.isdir(_fd_dir): + break + else: + _fd_dir = None + +# /dev/fd does not work on FreeBSD, see bug #478446 +if platform.system() in ('FreeBSD',) and _fd_dir == '/dev/fd': + _fd_dir = None + +if _fd_dir is not None: def get_open_fds(): - return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \ - if fd.isdigit()) + return (int(fd) for fd in os.listdir(_fd_dir) if fd.isdigit()) if platform.python_implementation() == 'PyPy': # EAGAIN observed with PyPy 1.8. @@ -46,6 +73,13 @@ if os.path.isdir("/proc/%i/fd" % os.getpid()): raise return range(max_fd_limit) +elif os.path.isdir("/proc/%s/fd" % os.getpid()): + # In order for this function to work in forked subprocesses, + # os.getpid() must be called from inside the function. + def get_open_fds(): + return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid()) + if fd.isdigit()) + else: def get_open_fds(): return range(max_fd_limit) @@ -62,7 +96,7 @@ fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): """ Spawns a bash shell running a specific commands - + @param mycommand: The command for bash to run @type mycommand: String @param debug: Turn bash debugging on (set -x) @@ -91,7 +125,7 @@ def spawn_autodep(mycommand, opt_name=None, **keywords): # Core part: tell the loader to preload logging library keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY - return spawn_bash(mycommand, opt_name=opt_name, **keywords) + return spawn_bash(mycommand, opt_name=opt_name, **keywords) def spawn_sandbox(mycommand, opt_name=None, **keywords): if not sandbox_capable: @@ -154,33 +188,31 @@ def run_exitfuncs(): atexit.register(run_exitfuncs) -# We need to make sure that any processes spawned are killed off when -# we exit. spawn() takes care of adding and removing pids to this list -# as it creates and cleans up processes. -spawned_pids = [] -def cleanup(): - while spawned_pids: - pid = spawned_pids.pop() +# It used to be necessary for API consumers to remove pids from spawned_pids, +# since otherwise it would accumulate a pids endlessly. Now, spawned_pids is +# just an empty dummy list, so for backward compatibility, ignore ValueError +# for removal on non-existent items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy spawned_pids. try: - # With waitpid and WNOHANG, only check the - # first element of the tuple since the second - # element may vary (bug #337465). - if os.waitpid(pid, os.WNOHANG)[0] == 0: - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - except OSError: - # This pid has been cleaned up outside - # of spawn(). + list.remove(self, item) + except ValueError: pass -atexit_register(cleanup) +spawned_pids = _dummy_list() + +def cleanup(): + pass def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, - path_lookup=True, pre_exec=None): + path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False, + unshare_ipc=False, cgroup=None): """ Spawns a given command. - + @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @@ -188,6 +220,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example + (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @@ -206,10 +239,19 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable - + @param close_fds: If True, then close all file descriptors except those + referenced by fd_pipes (default is True). + @type close_fds: Boolean + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String + logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) - + """ # mycommand is either a str or a list @@ -239,9 +281,9 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. @@ -269,21 +311,40 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, fd_pipes[1] = pw fd_pipes[2] = pw - pid = os.fork() + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if unshare_net or unshare_ipc: + find_library("c") - if pid == 0: - try: - _exec(binary, mycommand, opt_name, fd_pipes, - env, gid, groups, uid, umask, pre_exec) - except SystemExit: - raise - except Exception as e: - # We need to catch _any_ exception so that it doesn't - # propagate out of this function and cause exiting - # with anything other than os._exit() - sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) - traceback.print_exc() - sys.stderr.flush() + parent_pid = os.getpid() + pid = None + try: + pid = os.fork() + + if pid == 0: + try: + _exec(binary, mycommand, opt_name, fd_pipes, + env, gid, groups, uid, umask, pre_exec, close_fds, + unshare_net, unshare_ipc, cgroup) + except SystemExit: + raise + except Exception as e: + # We need to catch _any_ exception so that it doesn't + # propagate out of this function and cause exiting + # with anything other than os._exit() + writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), + noiselevel=-1) + traceback.print_exc() + sys.stderr.flush() + + finally: + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): @@ -291,7 +352,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # Add the pid to our local and the global pid lists. mypids.append(pid) - spawned_pids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. @@ -314,10 +374,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # and wait for it. retval = os.waitpid(pid, 0)[1] - # When it's done, we can remove it from the - # global pid list as well. - spawned_pids.remove(pid) - if retval: # If it failed, kill off anything else that # isn't dead yet. @@ -328,7 +384,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) - spawned_pids.remove(pid) # If it got a signal, return the signal that was sent. if (retval & 0xff): @@ -341,11 +396,11 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, return 0 def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, - pre_exec): + pre_exec, close_fds, unshare_net, unshare_ipc, cgroup): """ Execute a given binary with options - + @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @@ -366,10 +421,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ - + # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: @@ -384,6 +445,10 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, myargs = [opt_name] myargs.extend(mycommand[1:]) + # Avoid a potential UnicodeEncodeError from os.execve(). + myargs = [_unicode_encode(x, encoding=_encodings['fs'], + errors='strict') for x in myargs] + # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) @@ -396,15 +461,63 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) - _setup_pipes(fd_pipes) + _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) + + # Add to cgroup + # it's better to do it from the child since we can guarantee + # it is done before we start forking children + if cgroup: + with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: + f.write('%d\n' % os.getpid()) + + # Unshare (while still uid==0) + if unshare_net or unshare_ipc: + filename = find_library("c") + if filename is not None: + libc = LoadLibrary(filename) + if libc is not None: + CLONE_NEWIPC = 0x08000000 + CLONE_NEWNET = 0x40000000 + + flags = 0 + if unshare_net: + flags |= CLONE_NEWNET + if unshare_ipc: + flags |= CLONE_NEWIPC + + try: + if libc.unshare(flags) != 0: + writemsg("Unable to unshare: %s\n" % ( + errno.errorcode.get(ctypes.get_errno(), '?')), + noiselevel=-1) + else: + if unshare_net: + # 'up' the loopback + IFF_UP = 0x1 + ifreq = struct.pack('16sh', b'lo', IFF_UP) + SIOCSIFFLAGS = 0x8914 + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) + try: + fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) + except IOError as e: + writemsg("Unable to enable loopback interface: %s\n" % ( + errno.errorcode.get(e.errno, '?')), + noiselevel=-1) + sock.close() + except AttributeError: + # unshare() not supported by libc + pass # Set requested process permissions. if gid: - os.setgid(gid) + # Cast proxies to int, in case it matters. + os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: - os.setuid(uid) + # Cast proxies to int, in case it matters. + os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: @@ -413,9 +526,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # And switch to the new process. os.execve(binary, myargs, env) -def _setup_pipes(fd_pipes, close_fds=True): +def _setup_pipes(fd_pipes, close_fds=True, inheritable=None): """Setup pipes for a forked process. + Even when close_fds is False, file descriptors referenced as + values in fd_pipes are automatically closed if they do not also + occur as keys in fd_pipes. It is assumed that the caller will + explicitly add them to the fd_pipes keys if they are intended + to remain open. This allows for convenient elimination of + unnecessary duplicate file descriptors. + WARNING: When not followed by exec, the close_fds behavior can trigger interference from destructors that close file descriptors. This interference happens when the garbage @@ -426,22 +546,92 @@ def _setup_pipes(fd_pipes, close_fds=True): and also with CPython under some circumstances (as triggered by xmpppy in bug #374335). In order to close a safe subset of file descriptors, see portage.locks._close_fds(). + + NOTE: When not followed by exec, even when close_fds is False, + it's still possible for dup2() calls to cause interference in a + way that's similar to the way that close_fds interferes (since + dup2() has to close the target fd if it happens to be open). + It's possible to avoid such interference by using allocated + file descriptors as the keys in fd_pipes. For example: + + pr, pw = os.pipe() + fd_pipes[pw] = pw + + By using the allocated pw file descriptor as the key in fd_pipes, + it's not necessary for dup2() to close a file descriptor (it + actually does nothing in this case), which avoids possible + interference. """ - my_fds = {} + + reverse_map = {} # To protect from cases where direct assignment could - # clobber needed fds ({1:2, 2:1}) we first dupe the fds - # into unused fds. - for fd in fd_pipes: - my_fds[fd] = os.dup(fd_pipes[fd]) - # Then assign them to what they should be. - for fd in my_fds: - os.dup2(my_fds[fd], fd) + # clobber needed fds ({1:2, 2:1}) we create a reverse map + # in order to know when it's necessary to create temporary + # backup copies with os.dup(). + for newfd, oldfd in fd_pipes.items(): + newfds = reverse_map.get(oldfd) + if newfds is None: + newfds = [] + reverse_map[oldfd] = newfds + newfds.append(newfd) + + # Assign newfds via dup2(), making temporary backups when + # necessary, and closing oldfd if the caller has not + # explicitly requested for it to remain open by adding + # it to the keys of fd_pipes. + while reverse_map: + + oldfd, newfds = reverse_map.popitem() + old_fdflags = None + + for newfd in newfds: + if newfd in reverse_map: + # Make a temporary backup before re-assignment, assuming + # that backup_fd won't collide with a key in reverse_map + # (since all of the keys correspond to open file + # descriptors, and os.dup() only allocates a previously + # unused file discriptors). + backup_fd = os.dup(newfd) + reverse_map[backup_fd] = reverse_map.pop(newfd) + + if oldfd != newfd: + os.dup2(oldfd, newfd) + if _set_inheritable is not None: + # Don't do this unless _set_inheritable is available, + # since it's used below to ensure correct state, and + # otherwise /dev/null stdin fails to inherit (at least + # with Python versions from 3.1 to 3.3). + if old_fdflags is None: + old_fdflags = fcntl.fcntl(oldfd, fcntl.F_GETFD) + fcntl.fcntl(newfd, fcntl.F_SETFD, old_fdflags) + + if _set_inheritable is not None: + + inheritable_state = None + if not (old_fdflags is None or _FD_CLOEXEC is None): + inheritable_state = not bool(old_fdflags & _FD_CLOEXEC) + + if inheritable is not None: + if inheritable_state is not inheritable: + _set_inheritable(newfd, inheritable) + + elif newfd in (0, 1, 2): + if inheritable_state is not True: + _set_inheritable(newfd, True) + + if oldfd not in fd_pipes: + # If oldfd is not a key in fd_pipes, then it's safe + # to close now, since we've already made all of the + # requested duplicates. This also closes every + # backup_fd that may have been created on previous + # iterations of this loop. + os.close(oldfd) if close_fds: # Then close _all_ fds that haven't been explicitly # requested to be kept open. for fd in get_open_fds(): - if fd not in my_fds: + if fd not in fd_pipes: try: os.close(fd) except OSError: @@ -450,14 +640,22 @@ def _setup_pipes(fd_pipes, close_fds=True): def find_binary(binary): """ Given a binary name, find the binary in PATH - + @param binary: Name of the binary to find @type string @rtype: None or string @return: full path to binary or None if the binary could not be located. """ - for path in os.environ.get("PATH", "").split(":"): - filename = "%s/%s" % (path, binary) - if os.access(filename, os.X_OK) and os.path.isfile(filename): + paths = os.environ.get("PATH", "") + if sys.hexversion >= 0x3000000 and isinstance(binary, bytes): + # return bytes when input is bytes + paths = paths.encode(sys.getfilesystemencoding(), 'surrogateescape') + paths = paths.split(b':') + else: + paths = paths.split(':') + + for path in paths: + filename = _os.path.join(path, binary) + if _os.access(filename, os.X_OK) and _os.path.isfile(filename): return filename return None diff --git a/portage_with_autodep/pym/portage/process.pyo b/portage_with_autodep/pym/portage/process.pyo Binary files differindex c53af30..323d956 100644 --- a/portage_with_autodep/pym/portage/process.pyo +++ b/portage_with_autodep/pym/portage/process.pyo diff --git a/portage_with_autodep/pym/portage/proxy/__init__.pyo b/portage_with_autodep/pym/portage/proxy/__init__.pyo Binary files differindex b3d096b..69a2a54 100644 --- a/portage_with_autodep/pym/portage/proxy/__init__.pyo +++ b/portage_with_autodep/pym/portage/proxy/__init__.pyo diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.py b/portage_with_autodep/pym/portage/proxy/lazyimport.py index ad4a542..3057c05 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.py +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.py @@ -1,4 +1,4 @@ -# Copyright 2009 Gentoo Foundation +# Copyright 2009-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['lazyimport'] @@ -32,7 +32,7 @@ def _preload_portage_submodules(): while True: remaining = False for name in list(_module_proxies): - if name.startswith('portage.'): + if name.startswith('portage.') or name.startswith('_emerge.'): if name in imported: continue imported.add(name) diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo Binary files differindex 9da8089..cf4e80f 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.py b/portage_with_autodep/pym/portage/proxy/objectproxy.py index 92b36d1..a755774 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.py +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.py @@ -1,4 +1,4 @@ -# Copyright 2008-2009 Gentoo Foundation +# Copyright 2008-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -30,6 +30,13 @@ class ObjectProxy(object): result = object.__getattribute__(self, '_get_target')() return result(*args, **kwargs) + def __enter__(self): + return object.__getattribute__(self, '_get_target')().__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + return object.__getattribute__(self, '_get_target')().__exit__( + exc_type, exc_value, traceback) + def __setitem__(self, key, value): object.__getattribute__(self, '_get_target')()[key] = value diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo Binary files differindex f0919ff..a7c100a 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo diff --git a/portage_with_autodep/pym/portage/repository/__init__.pyo b/portage_with_autodep/pym/portage/repository/__init__.pyo Binary files differindex ab526c0..d98d5dc 100644 --- a/portage_with_autodep/pym/portage/repository/__init__.pyo +++ b/portage_with_autodep/pym/portage/repository/__init__.pyo diff --git a/portage_with_autodep/pym/portage/repository/config.py b/portage_with_autodep/pym/portage/repository/config.py index 20f1919..0d6edf4 100644 --- a/portage_with_autodep/pym/portage/repository/config.py +++ b/portage_with_autodep/pym/portage/repository/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import logging import warnings @@ -8,25 +10,34 @@ import sys import re try: - from configparser import ParsingError + from configparser import Error as ConfigParserError if sys.hexversion >= 0x3020000: from configparser import ConfigParser as SafeConfigParser else: from configparser import SafeConfigParser except ImportError: - from ConfigParser import SafeConfigParser, ParsingError + from ConfigParser import SafeConfigParser, Error as ConfigParserError +import portage from portage import eclass_cache, os from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, - REPO_NAME_LOC, USER_CONFIG_PATH) + PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH) +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.env.loaders import KeyValuePairFileLoader from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, - stack_lists, writemsg, writemsg_level) + stack_lists, writemsg, writemsg_level, _recursive_file_list) +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.localization import _ from portage import _unicode_decode from portage import _unicode_encode from portage import _encodings from portage import manifest +if sys.hexversion >= 0x3000000: + basestring = str + +# Characters prohibited by repoman's file.name check. +_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') + _valid_profile_formats = frozenset( ['pms', 'portage-1', 'portage-2']) @@ -48,38 +59,76 @@ def _gen_valid_repo(name): name = None return name +def _find_invalid_path_char(path, pos=0, endpos=None): + """ + Returns the position of the first invalid character found in basename, + or -1 if no invalid characters are found. + """ + if endpos is None: + endpos = len(path) + + m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos) + if m is not None: + return m.start() + + return -1 + class RepoConfig(object): """Stores config of one repository""" __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual', 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', - 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', + 'eclass_db', 'eclass_locations', 'eclass_overrides', + 'find_invalid_path_char', 'force', 'format', 'local_config', 'location', 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', - 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', - 'thin_manifest', 'update_changelog', 'user_location') + 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo', + 'sync_type', 'sync_uri', 'thin_manifest', 'update_changelog', + 'user_location', '_eapis_banned', '_eapis_deprecated', '_masters_orig') - def __init__(self, name, repo_opts): + def __init__(self, name, repo_opts, local_config=True): """Build a RepoConfig with options in repo_opts Try to read repo_name in repository location, but if it is not found use variable name as repository name""" - aliases = repo_opts.get('aliases') - if aliases is not None: - aliases = tuple(aliases.split()) + + force = repo_opts.get('force') + if force is not None: + force = tuple(force.split()) + self.force = force + if force is None: + force = () + + self.local_config = local_config + + if local_config or 'aliases' in force: + aliases = repo_opts.get('aliases') + if aliases is not None: + aliases = tuple(aliases.split()) + else: + aliases = None + self.aliases = aliases - eclass_overrides = repo_opts.get('eclass-overrides') - if eclass_overrides is not None: - eclass_overrides = tuple(eclass_overrides.split()) + if local_config or 'eclass-overrides' in force: + eclass_overrides = repo_opts.get('eclass-overrides') + if eclass_overrides is not None: + eclass_overrides = tuple(eclass_overrides.split()) + else: + eclass_overrides = None + self.eclass_overrides = eclass_overrides # Eclass databases and locations are computed later. self.eclass_db = None self.eclass_locations = None - # Masters from repos.conf override layout.conf. - masters = repo_opts.get('masters') - if masters is not None: - masters = tuple(masters.split()) + if local_config or 'masters' in force: + # Masters from repos.conf override layout.conf. + masters = repo_opts.get('masters') + if masters is not None: + masters = tuple(masters.split()) + else: + masters = None + self.masters = masters #The main-repo key makes only sense for the 'DEFAULT' section. @@ -93,11 +142,22 @@ class RepoConfig(object): priority = None self.priority = priority - sync = repo_opts.get('sync') - if sync is not None: - sync = sync.strip() - self.sync = sync + sync_cvs_repo = repo_opts.get('sync-cvs-repo') + if sync_cvs_repo is not None: + sync_cvs_repo = sync_cvs_repo.strip() + self.sync_cvs_repo = sync_cvs_repo or None + + sync_type = repo_opts.get('sync-type') + if sync_type is not None: + sync_type = sync_type.strip() + self.sync_type = sync_type or None + sync_uri = repo_opts.get('sync-uri') + if sync_uri is not None: + sync_uri = sync_uri.strip() + self.sync_uri = sync_uri or None + + # Not implemented. format = repo_opts.get('format') if format is not None: format = format.strip() @@ -106,7 +166,7 @@ class RepoConfig(object): location = repo_opts.get('location') self.user_location = location if location is not None and location.strip(): - if os.path.isdir(location): + if os.path.isdir(location) or portage._sync_disabled_warnings: location = os.path.realpath(location) else: location = None @@ -114,14 +174,23 @@ class RepoConfig(object): eapi = None missing = True + self.name = name if self.location is not None: eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC)) - name, missing = self._read_valid_repo_name(self.location) - elif name == "DEFAULT": + self.name, missing = self._read_valid_repo_name(self.location) + if missing: + # The name from repos.conf has to be used here for + # things like emerge-webrsync to work when the repo + # is empty (bug #484950). + if name is not None: + self.name = name + if portage._sync_disabled_warnings: + missing = False + + elif name == "DEFAULT": missing = False self.eapi = eapi - self.name = name self.missing_repo_name = missing # sign_commit is disabled by default, since it requires Git >=1.7.9, # and key_id configured by `git config user.signingkey key_id` @@ -137,18 +206,20 @@ class RepoConfig(object): self.cache_formats = None self.portage1_profiles = True self.portage1_profiles_compat = False + self.find_invalid_path_char = _find_invalid_path_char + self._masters_orig = None # Parse layout.conf. if self.location: - layout_filename = os.path.join(self.location, "metadata", "layout.conf") layout_data = parse_layout_conf(self.location, self.name)[0] + self._masters_orig = layout_data['masters'] # layout.conf masters may be overridden here if we have a masters # setting from the user's repos.conf if self.masters is None: self.masters = layout_data['masters'] - if layout_data['aliases']: + if (local_config or 'aliases' in force) and layout_data['aliases']: aliases = self.aliases if aliases is None: aliases = () @@ -156,6 +227,12 @@ class RepoConfig(object): # them the ability to do incremental overrides self.aliases = layout_data['aliases'] + tuple(aliases) + if layout_data['repo-name']: + # allow layout.conf to override repository name + # useful when having two copies of the same repo enabled + # to avoid modifying profiles/repo_name in one of them + self.name = layout_data['repo-name'] + for value in ('allow-missing-manifest', 'allow-provide-virtual', 'cache-formats', 'create-manifest', 'disable-manifest', 'manifest-hashes', @@ -163,9 +240,19 @@ class RepoConfig(object): 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): setattr(self, value.lower().replace("-", "_"), layout_data[value]) - self.portage1_profiles = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - self.portage1_profiles_compat = layout_data['profile-formats'] == ('portage-1-compat',) + self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + + self._eapis_banned = frozenset(layout_data['eapis-banned']) + self._eapis_deprecated = frozenset(layout_data['eapis-deprecated']) + + def eapi_is_banned(self, eapi): + return eapi in self._eapis_banned + + def eapi_is_deprecated(self, eapi): + return eapi in self._eapis_deprecated def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False): """ @@ -178,7 +265,11 @@ class RepoConfig(object): if not formats: if not force: return - formats = ('pms',) + # The default egencache format was 'pms' prior to portage-2.1.11.32 + # (portage versions prior to portage-2.1.11.14 will NOT + # recognize md5-dict format unless it is explicitly listed in + # layout.conf). + formats = ('md5-dict',) for fmt in formats: name = None @@ -209,7 +300,8 @@ class RepoConfig(object): kwds['hashes'] = self.manifest_hashes if self.disable_manifest: kwds['from_scratch'] = True - return manifest.Manifest(*args, **kwds) + kwds['find_invalid_path_char'] = self.find_invalid_path_char + return manifest.Manifest(*args, **portage._native_kwargs(kwds)) def update(self, new_repo): """Update repository with options in another RepoConfig""" @@ -272,8 +364,12 @@ class RepoConfig(object): repo_msg.append(indent + "format: " + self.format) if self.user_location: repo_msg.append(indent + "location: " + self.user_location) - if self.sync: - repo_msg.append(indent + "sync: " + self.sync) + if self.sync_cvs_repo: + repo_msg.append(indent + "sync-cvs-repo: " + self.sync_cvs_repo) + if self.sync_type: + repo_msg.append(indent + "sync-type: " + self.sync_type) + if self.sync_uri: + repo_msg.append(indent + "sync-uri: " + self.sync_uri) if self.masters: repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters)) if self.priority is not None: @@ -281,19 +377,19 @@ class RepoConfig(object): if self.aliases: repo_msg.append(indent + "aliases: " + " ".join(self.aliases)) if self.eclass_overrides: - repo_msg.append(indent + "eclass_overrides: " + \ + repo_msg.append(indent + "eclass-overrides: " + \ " ".join(self.eclass_overrides)) repo_msg.append("") return "\n".join(repo_msg) def __repr__(self): - return "<portage.repository.config.RepoConfig(name='%s', location='%s')>" % (self.name, _unicode_decode(self.location)) + return "<portage.repository.config.RepoConfig(name=%r, location=%r)>" % (self.name, _unicode_decode(self.location)) def __str__(self): d = {} for k in self.__slots__: d[k] = getattr(self, k, None) - return _unicode_decode("%s") % (d,) + return "%s" % (d,) if sys.hexversion < 0x3000000: @@ -306,11 +402,14 @@ class RepoConfigLoader(object): """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" @staticmethod - def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map): + def _add_repositories(portdir, portdir_overlay, prepos, + ignored_map, ignored_location_map, local_config, default_portdir): """Add overlays in PORTDIR_OVERLAY as repositories""" overlays = [] + portdir_orig = None if portdir: portdir = normalize_path(portdir) + portdir_orig = portdir overlays.append(portdir) try: port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] @@ -344,43 +443,57 @@ class RepoConfigLoader(object): #overlay priority is negative because we want them to be looked before any other repo base_priority = 0 for ov in overlays: - if os.path.isdir(ov): + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if isdir_raise_eaccess(ov) or \ + (base_priority == 0 and ov is portdir): repo_opts = default_repo_opts.copy() repo_opts['location'] = ov - repo = RepoConfig(None, repo_opts) + repo = RepoConfig(None, repo_opts, local_config=local_config) # repos_conf_opts contains options from repos.conf repos_conf_opts = repos_conf.get(repo.name) if repos_conf_opts is not None: # Selectively copy only the attributes which # repos.conf is allowed to override. - for k in ('aliases', 'eclass_overrides', 'masters', 'priority'): + for k in ('aliases', 'eclass_overrides', 'force', 'masters', + 'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'): v = getattr(repos_conf_opts, k, None) if v is not None: setattr(repo, k, v) if repo.name in prepos: + # Silently ignore when PORTDIR overrides the location + # setting from the default repos.conf (bug #478544). old_location = prepos[repo.name].location - if old_location is not None and old_location != repo.location: + if old_location is not None and \ + old_location != repo.location and \ + not (base_priority == 0 and + old_location == default_portdir): ignored_map.setdefault(repo.name, []).append(old_location) ignored_location_map[old_location] = repo.name if old_location == portdir: portdir = repo.user_location - if ov == portdir and portdir not in port_ov: - repo.priority = -1000 - elif repo.priority is None: - repo.priority = base_priority - base_priority += 1 + if repo.priority is None: + if base_priority == 0 and ov == portdir_orig: + # If it's the original PORTDIR setting and it's not + # in PORTDIR_OVERLAY, then it will be assigned a + # special priority setting later. + pass + else: + repo.priority = base_priority + base_priority += 1 prepos[repo.name] = repo else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) + + if not portage._sync_disabled_warnings: + writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1) return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): """Parse files in paths to load config""" parser = SafeConfigParser() @@ -388,49 +501,78 @@ class RepoConfigLoader(object): try: # Python >=3.2 read_file = parser.read_file + source_kwarg = 'source' except AttributeError: read_file = parser.readfp + source_kwarg = 'filename' + recursive_paths = [] for p in paths: - f = None - try: - f = io.open(_unicode_encode(p, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass + if isinstance(p, basestring): + recursive_paths.extend(_recursive_file_list(p)) else: + recursive_paths.append(p) + + for p in recursive_paths: + if isinstance(p, basestring): + f = None try: - read_file(f) - except ParsingError as e: - writemsg(_unicode_decode( - _("!!! Error while reading repo config file: %s\n") - ) % e, noiselevel=-1) - finally: - if f is not None: - f.close() - - prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults()) + f = io.open(_unicode_encode(p, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + else: + # The 'source' keyword argument is needed since otherwise + # ConfigParser in Python <3.3.3 may throw a TypeError + # because it assumes that f.name is a native string rather + # than binary when constructing error messages. + kwargs = {source_kwarg: p} + read_file(f, **portage._native_kwargs(kwargs)) + finally: + if f is not None: + f.close() + elif isinstance(p, io.StringIO): + kwargs = {source_kwarg: "<io.StringIO>"} + read_file(p, **portage._native_kwargs(kwargs)) + else: + raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p)) + + prepos['DEFAULT'] = RepoConfig("DEFAULT", + parser.defaults(), local_config=local_config) + for sname in parser.sections(): optdict = {} for oname in parser.options(sname): optdict[oname] = parser.get(sname, oname) - repo = RepoConfig(sname, optdict) - if repo.location and not os.path.exists(repo.location): - writemsg(_("!!! Invalid repos.conf entry '%s'" - " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1) + repo = RepoConfig(sname, optdict, local_config=local_config) + + if repo.sync_type is not None and repo.sync_uri is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") % + sname, level=logging.ERROR, noiselevel=-1) continue - if repo.name in prepos: - old_location = prepos[repo.name].location - if old_location is not None and repo.location is not None and old_location != repo.location: - ignored_map.setdefault(repo.name, []).append(old_location) - ignored_location_map[old_location] = repo.name - prepos[repo.name].update(repo) - else: - prepos[repo.name] = repo + if repo.sync_uri is not None and repo.sync_type is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type not in (None, "cvs", "git", "rsync"): + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") % + (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type == "cvs" and repo.sync_cvs_repo is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + # For backward compatibility with locations set via PORTDIR and + # PORTDIR_OVERLAY, delay validation of the location and repo.name + # until after PORTDIR and PORTDIR_OVERLAY have been processed. + prepos[sname] = repo def __init__(self, paths, settings): """Load config from files in paths""" @@ -441,15 +583,42 @@ class RepoConfigLoader(object): ignored_map = {} ignored_location_map = {} - portdir = settings.get('PORTDIR', '') - portdir_overlay = settings.get('PORTDIR_OVERLAY', '') + if "PORTAGE_REPOSITORIES" in settings: + portdir = "" + portdir_overlay = "" + portdir_sync = "" + else: + portdir = settings.get("PORTDIR", "") + portdir_overlay = settings.get("PORTDIR_OVERLAY", "") + portdir_sync = settings.get("SYNC", "") - self._parse(paths, prepos, ignored_map, ignored_location_map) + try: + self._parse(paths, prepos, ignored_map, + ignored_location_map, settings.local_config, + portdir) + except ConfigParserError as e: + writemsg( + _("!!! Error while reading repo config file: %s\n") % e, + noiselevel=-1) + # The configparser state is unreliable (prone to quirky + # exceptions) after it has thrown an error, so use empty + # config and try to fall back to PORTDIR{,_OVERLAY}. + prepos.clear() + prepos['DEFAULT'] = RepoConfig('DEFAULT', + {}, local_config=settings.local_config) + location_map.clear() + treemap.clear() + ignored_map.clear() + ignored_location_map.clear() + + default_portdir = os.path.join(os.sep, + settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage') # If PORTDIR_OVERLAY contains a repo with the same repo_name as # PORTDIR, then PORTDIR is overridden. portdir = self._add_repositories(portdir, portdir_overlay, prepos, - ignored_map, ignored_location_map) + ignored_map, ignored_location_map, settings.local_config, + default_portdir) if portdir and portdir.strip(): portdir = os.path.realpath(portdir) @@ -460,9 +629,51 @@ class RepoConfigLoader(object): for repo in prepos.values() if repo.location is not None and repo.missing_repo_name) - #Take aliases into account. - new_prepos = {} - for repo_name, repo in prepos.items(): + # Do this before expanding aliases, so that location_map and + # treemap consistently map unaliased names whenever available. + for repo_name, repo in list(prepos.items()): + if repo.location is None: + if repo_name != 'DEFAULT': + # Skip this warning for repoman (bug #474578). + if settings.local_config and paths: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf is missing location attribute") % + repo.name, level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + else: + if not portage._sync_disabled_warnings: + if not isdir_raise_eaccess(repo.location): + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has location attribute set " + "to nonexistent directory: '%s'") % + (repo_name, repo.location), level=logging.ERROR, noiselevel=-1) + + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if repo.name != 'gentoo': + del prepos[repo_name] + continue + + # After removing support for PORTDIR_OVERLAY, the following check can be: + # if repo.missing_repo_name: + if repo.missing_repo_name and repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf refers to repository " + "without repository name set in '%s'") % + (repo_name, os.path.join(repo.location, REPO_NAME_LOC)), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + if repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has name different " + "from repository name '%s' set inside repository") % + (repo_name, repo.name), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + location_map[repo.location] = repo_name + treemap[repo_name] = repo.location + + # Add alias mappings, but never replace unaliased mappings. + for repo_name, repo in list(prepos.items()): names = set() names.add(repo_name) if repo.aliases: @@ -470,36 +681,55 @@ class RepoConfigLoader(object): names.update(aliases) for name in names: - if name in new_prepos: + if name in prepos and prepos[name].location is not None: + if name == repo_name: + # unaliased names already handled earlier + continue writemsg_level(_("!!! Repository name or alias '%s', " + \ "defined for repository '%s', overrides " + \ "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1) - new_prepos[name] = repo - prepos = new_prepos + # Never replace an unaliased mapping with + # an aliased mapping. + continue + prepos[name] = repo + if repo.location is not None: + if repo.location not in location_map: + # Never replace an unaliased mapping with + # an aliased mapping. + location_map[repo.location] = name + treemap[name] = repo.location + + main_repo = prepos['DEFAULT'].main_repo + if main_repo is None or main_repo not in prepos: + #setting main_repo if it was not set in repos.conf + main_repo = location_map.get(portdir) + if main_repo is not None: + prepos['DEFAULT'].main_repo = main_repo + else: + prepos['DEFAULT'].main_repo = None + if portdir and not portage._sync_disabled_warnings: + writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1) - for (name, r) in prepos.items(): - if r.location is not None: - location_map[r.location] = name - treemap[name] = r.location + if main_repo is not None and prepos[main_repo].priority is None: + # This happens if main-repo has been set in repos.conf. + prepos[main_repo].priority = -1000 - # filter duplicates from aliases, by only including - # items where repo.name == key + # Backward compatible SYNC support for mirrorselect. + if portdir_sync and main_repo is not None: + if portdir_sync.startswith("rsync://"): + prepos[main_repo].sync_uri = portdir_sync + prepos[main_repo].sync_type = "rsync" - prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0) + # Include repo.name in sort key, for predictable sorting + # even when priorities are equal. + prepos_order = sorted(prepos.items(), + key=lambda r:(r[1].priority or 0, r[1].name)) + # filter duplicates from aliases, by only including + # items where repo.name == key prepos_order = [repo.name for (key, repo) in prepos_order - if repo.name == key and repo.location is not None] - - if prepos['DEFAULT'].main_repo is None or \ - prepos['DEFAULT'].main_repo not in prepos: - #setting main_repo if it was not set in repos.conf - if portdir in location_map: - prepos['DEFAULT'].main_repo = location_map[portdir] - elif portdir in ignored_location_map: - prepos['DEFAULT'].main_repo = ignored_location_map[portdir] - else: - prepos['DEFAULT'].main_repo = None - writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty. \n"), noiselevel=-1) + if repo.name == key and key != 'DEFAULT' and + repo.location is not None] self.prepos = prepos self.prepos_order = prepos_order @@ -578,6 +808,17 @@ class RepoConfigLoader(object): eclass_db.append(tree_db) repo.eclass_db = eclass_db + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + + if repo._masters_orig is None and self.mainRepo() and \ + repo.name != self.mainRepo().name and not portage._sync_disabled_warnings: + writemsg_level("!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") % + (repo.name, os.path.join(repo.location, "metadata", "layout.conf")) + + "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility") % + self.mainRepo().name, level=logging.WARNING, noiselevel=-1) + self._prepos_changed = True self._repo_location_list = [] @@ -613,10 +854,10 @@ class RepoConfigLoader(object): def mainRepo(self): """Returns the main repo""" - maid_repo = self.prepos['DEFAULT'].main_repo - if maid_repo is None: + main_repo = self.prepos['DEFAULT'].main_repo + if main_repo is None: return None - return self.prepos[maid_repo] + return self.prepos[main_repo] def _check_locations(self): """Check if repositories location are correct and show a warning message if not""" @@ -625,7 +866,7 @@ class RepoConfigLoader(object): if r.location is None: writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1) else: - if not os.path.isdir(r.location): + if not isdir_raise_eaccess(r.location) and not portage._sync_disabled_warnings: self.prepos_order.remove(name) writemsg(_("!!! Invalid Repository Location" " (not a dir): '%s'\n") % r.location, noiselevel=-1) @@ -650,19 +891,66 @@ class RepoConfigLoader(object): def get_repo_for_location(self, location): return self.prepos[self.get_name_for_location(location)] + def __setitem__(self, repo_name, repo): + # self.prepos[repo_name] = repo + raise NotImplementedError + def __getitem__(self, repo_name): return self.prepos[repo_name] + def __delitem__(self, repo_name): + if repo_name == self.prepos['DEFAULT'].main_repo: + self.prepos['DEFAULT'].main_repo = None + location = self.prepos[repo_name].location + del self.prepos[repo_name] + if repo_name in self.prepos_order: + self.prepos_order.remove(repo_name) + for k, v in self.location_map.copy().items(): + if v == repo_name: + del self.location_map[k] + if repo_name in self.treemap: + del self.treemap[repo_name] + self._repo_location_list = tuple(x for x in self._repo_location_list if x != location) + def __iter__(self): for repo_name in self.prepos_order: yield self.prepos[repo_name] -def load_repository_config(settings): - #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")] + def __contains__(self, repo_name): + return repo_name in self.prepos + + def config_string(self): + str_or_int_keys = ("format", "location", "main_repo", "priority", "sync_cvs_repo", "sync_type", "sync_uri") + str_tuple_keys = ("aliases", "eclass_overrides", "force") + repo_config_tuple_keys = ("masters",) + keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys + config_string = "" + for repo_name, repo in sorted(self.prepos.items()): + config_string += "\n[%s]\n" % repo_name + for key in sorted(keys): + if key == "main_repo" and repo_name != "DEFAULT": + continue + if getattr(repo, key) is not None: + if key in str_or_int_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), getattr(repo, key)) + elif key in str_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(getattr(repo, key))) + elif key in repo_config_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(x.name for x in getattr(repo, key))) + return config_string.lstrip("\n") + +def load_repository_config(settings, extra_files=None): repoconfigpaths = [] - if settings.local_config: - repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], - USER_CONFIG_PATH, "repos.conf")) + if "PORTAGE_REPOSITORIES" in settings: + repoconfigpaths.append(io.StringIO(settings["PORTAGE_REPOSITORIES"])) + else: + if portage._working_copy: + repoconfigpaths.append(os.path.join(PORTAGE_BASE_PATH, "cnf", "repos.conf")) + else: + repoconfigpaths.append(os.path.join(settings.global_config_path, "repos.conf")) + repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf")) + if extra_files: + repoconfigpaths.extend(extra_files) return RepoConfigLoader(repoconfigpaths, settings) def _get_repo_name(repo_location, cached=None): @@ -696,6 +984,9 @@ def parse_layout_conf(repo_location, repo_name=None): data['allow-provide-virtual'] = \ layout_data.get('allow-provide-virtuals', 'false').lower() == 'true' + data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split()) + data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split()) + data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \ == 'true' @@ -705,6 +996,8 @@ def parse_layout_conf(repo_location, repo_name=None): data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \ == 'true' + data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', '')) + manifest_policy = layout_data.get('use-manifests', 'strict').lower() data['allow-missing-manifest'] = manifest_policy != 'strict' data['create-manifest'] = manifest_policy != 'false' @@ -712,10 +1005,19 @@ def parse_layout_conf(repo_location, repo_name=None): # for compatibility w/ PMS, fallback to pms; but also check if the # cache exists or not. - cache_formats = layout_data.get('cache-formats', 'pms').lower().split() - if 'pms' in cache_formats and not os.path.isdir( - os.path.join(repo_location, 'metadata', 'cache')): - cache_formats.remove('pms') + cache_formats = layout_data.get('cache-formats', '').lower().split() + if not cache_formats: + # Auto-detect cache formats, and prefer md5-cache if available. + # This behavior was deployed in portage-2.1.11.14, so that the + # default egencache format could eventually be changed to md5-dict + # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14 + # will NOT recognize md5-dict format unless it is explicitly + # listed in layout.conf. + cache_formats = [] + if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')): + cache_formats.append('md5-dict') + if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')): + cache_formats.append('pms') data['cache-formats'] = tuple(cache_formats) manifest_hashes = layout_data.get('manifest-hashes') @@ -754,7 +1056,7 @@ def parse_layout_conf(repo_location, repo_name=None): raw_formats = layout_data.get('profile-formats') if raw_formats is None: - if eapi in ('4-python',): + if eapi_allows_directories_on_profile_level_and_repository_level(eapi): raw_formats = ('portage-1',) else: raw_formats = ('portage-1-compat',) diff --git a/portage_with_autodep/pym/portage/repository/config.pyo b/portage_with_autodep/pym/portage/repository/config.pyo Binary files differindex f9ee26d..b289c2c 100644 --- a/portage_with_autodep/pym/portage/repository/config.pyo +++ b/portage_with_autodep/pym/portage/repository/config.pyo diff --git a/portage_with_autodep/pym/portage/tests/__init__.py b/portage_with_autodep/pym/portage/tests/__init__.py index 492ece4..84e732a 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.py +++ b/portage_with_autodep/pym/portage/tests/__init__.py @@ -1,5 +1,5 @@ # tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -7,26 +7,40 @@ from __future__ import print_function import sys import time import unittest -from optparse import OptionParser, OptionValueError try: from unittest.runner import _TextTestResult # new in python-2.7 except ImportError: from unittest import _TextTestResult +try: + # They added the skip framework to python-2.7. + # Drop this once we drop python-2.6 support. + unittest_skip_shims = False + import unittest.SkipTest as SkipTest # new in python-2.7 +except ImportError: + unittest_skip_shims = True + +import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage.util._argparse import ArgumentParser def main(): suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) - parser = OptionParser(usage=usage) - parser.add_option("-l", "--list", help="list all tests", + parser = ArgumentParser(usage=usage) + parser.add_argument("-l", "--list", help="list all tests", action="store_true", dest="list_tests") - (options, args) = parser.parse_args(args=sys.argv) + options, args = parser.parse_known_args(args=sys.argv) + + if (os.environ.get('NOCOLOR') in ('yes', 'true') or + os.environ.get('TERM') == 'dumb' or + not sys.stdout.isatty()): + portage.output.nocolor() if options.list_tests: testdir = os.path.dirname(sys.argv[0]) @@ -70,15 +84,12 @@ def getTestFromCommandLine(args, base_path): def getTestDirs(base_path): TEST_FILE = b'__test__' - svn_dirname = b'.svn' testDirs = [] # the os.walk help mentions relative paths as being quirky # I was tired of adding dirs to the list, so now we add __test__ # to each dir we want tested. for root, dirs, files in os.walk(base_path): - if svn_dirname in dirs: - dirs.remove(svn_dirname) try: root = _unicode_decode(root, encoding=_encodings['fs'], errors='strict') @@ -93,7 +104,7 @@ def getTestDirs(base_path): def getTestNames(path): files = os.listdir(path) - files = [ f[:-3] for f in files if f.startswith("test") and f.endswith(".py") ] + files = [f[:-3] for f in files if f.startswith("test") and f.endswith(".py")] files.sort() return files @@ -134,14 +145,14 @@ class TextTestResult(_TextTestResult): self.portage_skipped = [] def addTodo(self, test, info): - self.todoed.append((test,info)) + self.todoed.append((test, info)) if self.showAll: self.stream.writeln("TODO") elif self.dots: self.stream.write(".") def addPortageSkip(self, test, info): - self.portage_skipped.append((test,info)) + self.portage_skipped.append((test, info)) if self.showAll: self.stream.writeln("SKIP") elif self.dots: @@ -185,10 +196,14 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) return + ok = False try: testMethod() ok = True + except SkipTest as e: + result.addPortageSkip(self, "%s: SKIP: %s" % + (testMethod, str(e))) except self.failureException: if self.portage_skip is not None: if self.portage_skip is True: @@ -197,13 +212,14 @@ class TestCase(unittest.TestCase): result.addPortageSkip(self, "%s: SKIP: %s" % (testMethod, self.portage_skip)) elif self.todo: - result.addTodo(self,"%s: TODO" % testMethod) + result.addTodo(self, "%s: TODO" % testMethod) else: result.addFailure(self, sys.exc_info()) except (KeyboardInterrupt, SystemExit): raise except: result.addError(self, sys.exc_info()) + try: self.tearDown() except SystemExit: @@ -213,7 +229,8 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) ok = False - if ok: result.addSuccess(self) + if ok: + result.addSuccess(self) finally: result.stopTest(self) @@ -230,10 +247,48 @@ class TestCase(unittest.TestCase): except excClass: return else: - if hasattr(excClass,'__name__'): excName = excClass.__name__ + if hasattr(excClass, '__name__'): excName = excClass.__name__ else: excName = str(excClass) raise self.failureException("%s not raised: %s" % (excName, msg)) + def assertExists(self, path): + """Make sure |path| exists""" + if not os.path.exists(path): + msg = ['path is missing: %s' % (path,)] + while path != '/': + path = os.path.dirname(path) + if not path: + # If we're given something like "foo", abort once we get to "". + break + result = os.path.exists(path) + msg.append('\tos.path.exists(%s): %s' % (path, result)) + if result: + msg.append('\tcontents: %r' % os.listdir(path)) + break + raise self.failureException('\n'.join(msg)) + + def assertNotExists(self, path): + """Make sure |path| does not exist""" + if os.path.exists(path): + raise self.failureException('path exists when it should not: %s' % path) + +if unittest_skip_shims: + # Shim code for <python-2.7. + class SkipTest(Exception): + """unittest.SkipTest shim for <python-2.7""" + + def skipTest(self, reason): + raise SkipTest(reason) + setattr(TestCase, 'skipTest', skipTest) + + def assertIn(self, member, container, msg=None): + self.assertTrue(member in container, msg=msg) + setattr(TestCase, 'assertIn', assertIn) + + def assertNotIn(self, member, container, msg=None): + self.assertFalse(member in container, msg=msg) + setattr(TestCase, 'assertNotIn', assertNotIn) + class TextTestRunner(unittest.TextTestRunner): """ We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable @@ -271,8 +326,8 @@ class TextTestRunner(unittest.TextTestRunner): self.stream.writeln("OK") return result -test_cps = ['sys-apps/portage','virtual/portage'] -test_versions = ['1.0', '1.0-r1','2.3_p4','1.0_alpha57'] -test_slots = [ None, '1','gentoo-sources-2.6.17','spankywashere'] -test_usedeps = ['foo','-bar', ('foo','bar'), - ('foo','-bar'), ('foo?', '!bar?') ] +test_cps = ['sys-apps/portage', 'virtual/portage'] +test_versions = ['1.0', '1.0-r1', '2.3_p4', '1.0_alpha57'] +test_slots = [None, '1', 'gentoo-sources-2.6.17', 'spankywashere'] +test_usedeps = ['foo', '-bar', ('foo', 'bar'), + ('foo', '-bar'), ('foo?', '!bar?')] diff --git a/portage_with_autodep/pym/portage/tests/__init__.pyo b/portage_with_autodep/pym/portage/tests/__init__.pyo Binary files differindex 0e961b8..aa0215b 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo Binary files differindex a1241e5..2a1215a 100644 --- a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py index aef8d74..fdbb6fe 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py @@ -1,20 +1,26 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from itertools import chain import stat +import subprocess +import sys -from portage.const import BASH_BINARY, PORTAGE_BIN_PATH +from portage.const import BASH_BINARY, PORTAGE_BASE_PATH, PORTAGE_BIN_PATH from portage.tests import TestCase from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings -from portage import _shell_quote from portage import _unicode_decode, _unicode_encode class BashSyntaxTestCase(TestCase): def testBashSyntax(self): - for parent, dirs, files in os.walk(PORTAGE_BIN_PATH): + locations = [PORTAGE_BIN_PATH] + misc_dir = os.path.join(PORTAGE_BASE_PATH, "misc") + if os.path.isdir(misc_dir): + locations.append(misc_dir) + for parent, dirs, files in \ + chain.from_iterable(os.walk(x) for x in locations): parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') for x in files: @@ -36,7 +42,13 @@ class BashSyntaxTestCase(TestCase): f.close() if line[:2] == '#!' and \ 'bash' in line: - cmd = "%s -n %s" % (_shell_quote(BASH_BINARY), _shell_quote(x)) - status, output = subprocess_getstatusoutput(cmd) + cmd = [BASH_BINARY, "-n", x] + cmd = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], + encoding=_encodings['fs']) + status = proc.wait() self.assertEqual(os.WIFEXITED(status) and \ os.WEXITSTATUS(status) == os.EX_OK, True, msg=output) diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo Binary files differindex a7ddc80..b066527 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py index f90a666..1d44e68 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py @@ -1,4 +1,4 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import itertools @@ -10,8 +10,6 @@ from portage import os from portage import _encodings from portage import _unicode_decode, _unicode_encode -import py_compile - class CompileModulesTestCase(TestCase): def testCompileModules(self): @@ -34,13 +32,13 @@ class CompileModulesTestCase(TestCase): do_compile = True else: # Check for python shebang - f = open(_unicode_encode(x, - encoding=_encodings['fs'], errors='strict'), 'rb') - line = _unicode_decode(f.readline(), - encoding=_encodings['content'], errors='replace') - f.close() - if line[:2] == '#!' and \ - 'python' in line: + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + line = _unicode_decode(f.readline(), + encoding=_encodings['content'], errors='replace') + if line[:2] == '#!' and 'python' in line: do_compile = True if do_compile: - py_compile.compile(x, cfile='/dev/null', doraise=True) + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + compile(f.read(), x, 'exec') diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo Binary files differindex 7b1460d..8e32e1f 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py index 8d257c5..34261f4 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.const import PORTAGE_PYM_PATH diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo Binary files differindex b3a1d61..b566748 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/runTests b/portage_with_autodep/pym/portage/tests/runTests index 4c10087..60bcf31 100755 --- a/portage_with_autodep/pym/portage/tests/runTests +++ b/portage_with_autodep/pym/portage/tests/runTests @@ -1,18 +1,25 @@ #!/usr/bin/python -Wd # runTests.py -- Portage Unit Test Functionality -# Copyright 2006 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os, sys import os.path as osp import grp +import platform import pwd import signal def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Pretend that the current user's uid/gid are the 'portage' uid/gid, # so things go smoothly regardless of the current user and global @@ -22,23 +29,33 @@ os.environ["PORTAGE_GRPNAME"] = grp.getgrgid(os.getgid()).gr_name # Insert our parent dir so we can do shiny import "tests" # This line courtesy of Marienz and Pkgcore ;) -sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.abspath(__file__))))) +sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__))))) import portage +portage._internal_caller = True # Ensure that we don't instantiate portage.settings, so that tests should # work the same regardless of global configuration file state/existence. portage._disable_legacy_globals() +if os.environ.get('NOCOLOR') in ('yes', 'true'): + portage.output.nocolor() + import portage.tests as tests from portage.const import PORTAGE_BIN_PATH path = os.environ.get("PATH", "").split(":") path = [x for x in path if x] -if not path or not os.path.samefile(path[0], PORTAGE_BIN_PATH): + +insert_bin_path = True +try: + insert_bin_path = not path or \ + not os.path.samefile(path[0], PORTAGE_BIN_PATH) +except OSError: + pass + +if insert_bin_path: path.insert(0, PORTAGE_BIN_PATH) os.environ["PATH"] = ":".join(path) -del path - if __name__ == "__main__": sys.exit(tests.main()) diff --git a/portage_with_autodep/pym/portage/update.py b/portage_with_autodep/pym/portage/update.py index 34e4663..92aba9a 100644 --- a/portage_with_autodep/pym/portage/update.py +++ b/portage_with_autodep/pym/portage/update.py @@ -1,11 +1,14 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re import stat import sys +import warnings from portage import os from portage import _encodings @@ -13,50 +16,107 @@ from portage import _unicode_decode from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.dep:Atom,dep_getkey,isvalidatom,' + \ - 'remove_slot', + 'portage.dep:Atom,dep_getkey,isvalidatom,match_from_list', 'portage.util:ConfigProtect,new_protect_filename,' + \ 'normalize_path,write_atomic,writemsg', - 'portage.util.listdir:_ignorecvs_dirs', - 'portage.versions:ververify' + 'portage.versions:_get_slot_re', ) -from portage.const import USER_CONFIG_PATH +from portage.const import USER_CONFIG_PATH, VCS_DIRS +from portage.eapi import _get_eapi_attrs from portage.exception import DirectoryNotFound, InvalidAtom, PortageException from portage.localization import _ if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode ignored_dbentries = ("CONTENTS", "environment.bz2") -def update_dbentry(update_cmd, mycontent): +def update_dbentry(update_cmd, mycontent, eapi=None, parent=None): + + if parent is not None: + eapi = parent.eapi + if update_cmd[0] == "move": - old_value = str(update_cmd[1]) - if old_value in mycontent: - new_value = str(update_cmd[2]) - old_value = re.escape(old_value); - mycontent = re.sub(old_value+"(:|$|\\s)", new_value+"\\1", mycontent) - def myreplace(matchobj): - # Strip slot and * operator if necessary - # so that ververify works. - ver = remove_slot(matchobj.group(2)) - ver = ver.rstrip("*") - if ververify(ver): - return "%s-%s" % (new_value, matchobj.group(2)) - else: - return "".join(matchobj.groups()) - mycontent = re.sub("(%s-)(\\S*)" % old_value, myreplace, mycontent) + old_value = _unicode(update_cmd[1]) + new_value = _unicode(update_cmd[2]) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if old_value in mycontent and isvalidatom(new_value, eapi=eapi): + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if old_value not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != old_value: + continue + + new_atom = Atom(token.replace(old_value, new_value, 1), + eapi=eapi) + + # Avoid creating self-blockers for bug #367215. + if new_atom.blocker and parent is not None and \ + parent.cp == new_atom.cp and \ + match_from_list(new_atom, [parent]): + continue + + split_content[i] = _unicode(new_atom) + modified = True + + if modified: + mycontent = "".join(split_content) + elif update_cmd[0] == "slotmove" and update_cmd[1].operator is None: - pkg, origslot, newslot = update_cmd[1:] - old_value = "%s:%s" % (pkg, origslot) - if old_value in mycontent: - old_value = re.escape(old_value) - new_value = "%s:%s" % (pkg, newslot) - mycontent = re.sub(old_value+"($|\\s)", new_value+"\\1", mycontent) + orig_atom, origslot, newslot = update_cmd[1:] + orig_cp = orig_atom.cp + + # We don't support versioned slotmove atoms here, since it can be + # difficult to determine if the version constraints really match + # the atoms that we're trying to update. + if orig_atom.version is None and orig_cp in mycontent: + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if orig_cp not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != orig_cp: + continue + if atom.slot is None or atom.slot != origslot: + continue + + slot_part = newslot + if atom.sub_slot is not None: + if atom.sub_slot == origslot: + sub_slot = newslot + else: + sub_slot = atom.sub_slot + slot_part += "/" + sub_slot + if atom.slot_operator is not None: + slot_part += atom.slot_operator + + split_content[i] = atom.with_slot(slot_part) + modified = True + + if modified: + mycontent = "".join(split_content) + return mycontent -def update_dbentries(update_iter, mydata): +def update_dbentries(update_iter, mydata, eapi=None, parent=None): """Performs update commands and returns a dict containing only the updated items.""" updated_items = {} @@ -70,7 +130,8 @@ def update_dbentries(update_iter, mydata): is_encoded = mycontent is not orig_content orig_content = mycontent for update_cmd in update_iter: - mycontent = update_dbentry(update_cmd, mycontent) + mycontent = update_dbentry(update_cmd, mycontent, + eapi=eapi, parent=parent) if mycontent != orig_content: if is_encoded: mycontent = _unicode_encode(mycontent, @@ -79,10 +140,14 @@ def update_dbentries(update_iter, mydata): updated_items[k] = mycontent return updated_items -def fixdbentries(update_iter, dbdir): +def fixdbentries(update_iter, dbdir, eapi=None, parent=None): """Performs update commands which result in search and replace operations for each of the files in dbdir (excluding CONTENTS and environment.bz2). Returns True when actual modifications are necessary and False otherwise.""" + + warnings.warn("portage.update.fixdbentries() is deprecated", + DeprecationWarning, stacklevel=2) + mydata = {} for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]: file_path = os.path.join(dbdir, myfile) @@ -91,7 +156,8 @@ def fixdbentries(update_iter, dbdir): mode='r', encoding=_encodings['repo.content'], errors='replace') as f: mydata[myfile] = f.read() - updated_items = update_dbentries(update_iter, mydata) + updated_items = update_dbentries(update_iter, mydata, + eapi=eapi, parent=parent) for myfile, mycontent in updated_items.items(): file_path = os.path.join(dbdir, myfile) write_atomic(file_path, mycontent, encoding=_encodings['repo.content']) @@ -143,6 +209,8 @@ def grab_updates(updpath, prev_mtimes=None): def parse_updates(mycontent): """Valid updates are returned as a list of split update commands.""" + eapi_attrs = _get_eapi_attrs(None) + slot_re = _get_slot_re(eapi_attrs) myupd = [] errors = [] mylines = mycontent.splitlines() @@ -194,12 +262,28 @@ def parse_updates(mycontent): errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue + invalid_slot = False + for slot in (origslot, newslot): + m = slot_re.match(slot) + if m is None: + invalid_slot = True + break + if "/" in slot: + # EAPI 4-slot-abi style SLOT is currently not supported. + invalid_slot = True + break + + if invalid_slot: + errors.append(_("ERROR: Malformed update entry '%s'") % myline) + continue + # The list of valid updates is filtered by continue statements above. myupd.append(mysplit) return myupd, errors def update_config_files(config_root, protect, protect_mask, update_iter, match_callback = None): - """Perform global updates on /etc/portage/package.*. + """Perform global updates on /etc/portage/package.*, /etc/portage/profile/package.*, + /etc/portage/profile/packages and /etc/portage/sets. config_root - location of files to update protect - list of paths from CONFIG_PROTECT protect_mask - list of paths from CONFIG_PROTECT_MASK @@ -222,9 +306,15 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c "package.accept_keywords", "package.env", "package.keywords", "package.license", "package.mask", "package.properties", - "package.unmask", "package.use" + "package.unmask", "package.use", "sets" ] - myxfiles += [os.path.join("profile", x) for x in myxfiles] + myxfiles += [os.path.join("profile", x) for x in ( + "packages", "package.accept_keywords", + "package.keywords", "package.mask", + "package.unmask", "package.use", + "package.use.force", "package.use.mask", + "package.use.stable.force", "package.use.stable.mask" + )] abs_user_config = os.path.join(config_root, USER_CONFIG_PATH) recursivefiles = [] for x in myxfiles: @@ -243,7 +333,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c except UnicodeDecodeError: dirs.remove(y_enc) continue - if y.startswith(".") or y in _ignorecvs_dirs: + if y.startswith(".") or y in VCS_DIRS: dirs.remove(y_enc) for y in files: try: @@ -273,7 +363,6 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c if f is not None: f.close() - # update /etc/portage/packages.* ignore_line_re = re.compile(r'^#|^\s*$') if repo_dict is None: update_items = [(None, update_iter)] @@ -293,6 +382,9 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c if atom[:1] == "-": # package.mask supports incrementals atom = atom[1:] + if atom[:1] == "*": + # packages file supports "*"-prefixed atoms as indication of system packages. + atom = atom[1:] if not isvalidatom(atom): continue new_atom = update_dbentry(update_cmd, atom) diff --git a/portage_with_autodep/pym/portage/update.pyo b/portage_with_autodep/pym/portage/update.pyo Binary files differindex 9628ea9..2979af7 100644 --- a/portage_with_autodep/pym/portage/update.pyo +++ b/portage_with_autodep/pym/portage/update.pyo diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py index 69bd58a..af4a4fe 100644 --- a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py @@ -61,18 +61,18 @@ def ExtractKernelVersion(base_dir): # Grab a list of files named localversion* and sort them localversions = os.listdir(base_dir) - for x in range(len(localversions)-1,-1,-1): + for x in range(len(localversions) - 1, -1, -1): if localversions[x][:12] != "localversion": del localversions[x] localversions.sort() # Append the contents of each to the version string, stripping ALL whitespace for lv in localversions: - version += "".join( " ".join( grabfile( base_dir+ "/" + lv ) ).split() ) + version += "".join(" ".join(grabfile(base_dir + "/" + lv)).split()) # Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace kernelconfig = getconfig(base_dir+"/.config") if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig: version += "".join(kernelconfig["CONFIG_LOCALVERSION"].split()) - return (version,None) + return (version, None) diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo Binary files differindex d0302fd..132515d 100644 --- a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo diff --git a/portage_with_autodep/pym/portage/util/SlotObject.py b/portage_with_autodep/pym/portage/util/SlotObject.py index a59dfc1..4bb6822 100644 --- a/portage_with_autodep/pym/portage/util/SlotObject.py +++ b/portage_with_autodep/pym/portage/util/SlotObject.py @@ -48,4 +48,3 @@ class SlotObject(object): setattr(obj, myattr, getattr(self, myattr)) return obj - diff --git a/portage_with_autodep/pym/portage/util/SlotObject.pyo b/portage_with_autodep/pym/portage/util/SlotObject.pyo Binary files differindex 11d0ec7..08133cb 100644 --- a/portage_with_autodep/pym/portage/util/SlotObject.pyo +++ b/portage_with_autodep/pym/portage/util/SlotObject.pyo diff --git a/portage_with_autodep/pym/portage/util/__init__.py b/portage_with_autodep/pym/portage/util/__init__.py index 2e0a32b..24553da 100644 --- a/portage_with_autodep/pym/portage/util/__init__.py +++ b/portage_with_autodep/pym/portage/util/__init__.py @@ -1,6 +1,8 @@ -# Copyright 2004-2012 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['apply_permissions', 'apply_recursive_permissions', 'apply_secpass_permissions', 'apply_stat_permissions', 'atomic_ofstream', 'cmp_sort_key', 'ConfigProtect', 'dump_traceback', 'ensure_dirs', @@ -31,21 +33,26 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'pickle', 'portage.dep:Atom', - 'portage.util.listdir:_ignorecvs_dirs' + 'subprocess', ) from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode +from portage.const import VCS_DIRS from portage.exception import InvalidAtom, PortageException, FileNotFound, \ OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict +if sys.hexversion >= 0x3000000: + _unicode = str +else: + _unicode = unicode + noiselimit = 0 def initialize_logger(level=logging.WARN): @@ -57,7 +64,7 @@ def initialize_logger(level=logging.WARN): """ logging.basicConfig(level=logging.WARN, format='[%(levelname)-4s] %(message)s') -def writemsg(mystr,noiselevel=0,fd=None): +def writemsg(mystr, noiselevel=0, fd=None): """Prints out warning and debug messages based on the noiselimit setting""" global noiselimit if fd is None: @@ -75,7 +82,7 @@ def writemsg(mystr,noiselevel=0,fd=None): fd.write(mystr) fd.flush() -def writemsg_stdout(mystr,noiselevel=0): +def writemsg_stdout(mystr, noiselevel=0): """Prints messages stdout based on the noiselimit setting""" writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout) @@ -100,7 +107,7 @@ def writemsg_level(msg, level=0, noiselevel=0): writemsg(msg, noiselevel=noiselevel, fd=fd) def normalize_path(mypath): - """ + """ os.path.normpath("//foo") returns "//foo" instead of "/foo" We dislike this behavior so we create our own normpath func to fix it. @@ -120,8 +127,8 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False """This function grabs the lines in a file, normalizes whitespace and returns lines in a list; if a line begins with a #, it is ignored, as are empty lines""" - mylines=grablines(myfilename, recursive, remember_source_file=True) - newlines=[] + mylines = grablines(myfilename, recursive, remember_source_file=True) + newlines = [] for x, source_file in mylines: #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line @@ -139,10 +146,10 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False myline = " ".join(myline) if not myline: continue - if myline[0]=="#": + if myline[0] == "#": # Check if we have a compat-level string. BC-integration data. # '##COMPAT==>N<==' 'some string attached to it' - mylinetest = myline.split("<==",1) + mylinetest = myline.split("<==", 1) if len(mylinetest) == 2: myline_potential = mylinetest[1] mylinetest = mylinetest[0].split("##COMPAT==>") @@ -159,7 +166,7 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False newlines.append(myline) return newlines -def map_dictlist_vals(func,myDict): +def map_dictlist_vals(func, myDict): """Performs a function on each value of each key in a dictlist. Returns a new dictlist.""" new_dl = {} @@ -173,7 +180,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 Stacks an array of dict-types into one array. Optionally merging or overwriting matching key/value pairs for the dict[key]->list. Returns a single dict. Higher index in lists is preferenced. - + Example usage: >>> from portage.util import stack_dictlist >>> print stack_dictlist( [{'a':'b'},{'x':'y'}]) @@ -188,7 +195,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 >>> { 'KEYWORDS':['alpha'] } >>> print stack_dictlist( [a,b], incrementals=['KEYWORDS']) >>> { 'KEYWORDS':['alpha'] } - + @param original_dicts a list of (dictionary objects or None) @type list @param incremental True or false depending on whether new keys should overwrite @@ -199,7 +206,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 @type list @param ignore_none Appears to be ignored, but probably was used long long ago. @type boolean - + """ final_dict = {} for mydict in original_dicts: @@ -208,7 +215,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 for y in mydict: if not y in final_dict: final_dict[y] = [] - + for thing in mydict[y]: if thing: if incremental or y in incrementals: @@ -245,12 +252,13 @@ def stack_dicts(dicts, incremental=0, incrementals=[], ignore_none=0): def append_repo(atom_list, repo_name, remember_source_file=False): """ Takes a list of valid atoms without repo spec and appends ::repo_name. + If an atom already has a repo part, then it is preserved (see bug #461948). """ if remember_source_file: - return [(Atom(atom + "::" + repo_name, allow_wildcard=True, allow_repo=True), source) \ + return [(atom.repo is not None and atom or atom.with_repo(repo_name), source) \ for atom, source in atom_list] else: - return [Atom(atom + "::" + repo_name, allow_wildcard=True, allow_repo=True) \ + return [atom.repo is not None and atom or atom.with_repo(repo_name) \ for atom in atom_list] def stack_lists(lists, incremental=1, remember_source_file=False, @@ -334,7 +342,7 @@ def stack_lists(lists, incremental=1, remember_source_file=False, def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): """ This function grabs the lines in a file, normalizes whitespace and returns lines in a dictionary - + @param myfilename: file to process @type myfilename: string (path) @param juststrings: only return strings @@ -350,9 +358,9 @@ def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): 1. Returns the lines in a file in a dictionary, for example: 'sys-apps/portage x86 amd64 ppc' would return - { "sys-apps/portage" : [ 'x86', 'amd64', 'ppc' ] + {"sys-apps/portage" : ['x86', 'amd64', 'ppc']} """ - newdict={} + newdict = {} for x in grablines(myfilename, recursive): #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line #into single spaces. @@ -379,52 +387,75 @@ def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): newdict[k] = " ".join(v) return newdict -def read_corresponding_eapi_file(filename): +_eapi_cache = {} + +def read_corresponding_eapi_file(filename, default="0"): """ Read the 'eapi' file from the directory 'filename' is in. Returns "0" if the file is not present or invalid. """ - default = "0" eapi_file = os.path.join(os.path.dirname(filename), "eapi") try: - f = io.open(_unicode_encode(eapi_file, + eapi = _eapi_cache[eapi_file] + except KeyError: + pass + else: + if eapi is None: + return default + return eapi + + eapi = None + try: + with io.open(_unicode_encode(eapi_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace') - lines = f.readlines() + mode='r', encoding=_encodings['repo.content'], errors='replace') as f: + lines = f.readlines() if len(lines) == 1: eapi = lines[0].rstrip("\n") else: writemsg(_("--- Invalid 'eapi' file (doesn't contain exactly one line): %s\n") % (eapi_file), noiselevel=-1) - eapi = default - f.close() except IOError: - eapi = default + pass + _eapi_cache[eapi_file] = eapi + if eapi is None: + return default return eapi def grabdict_package(myfilename, juststrings=0, recursive=0, allow_wildcard=False, allow_repo=False, verify_eapi=False, eapi=None): """ Does the same thing as grabdict except it validates keys with isvalidatom()""" - pkgs=grabdict(myfilename, juststrings, empty=1, recursive=recursive) - if not pkgs: - return pkgs - if verify_eapi and eapi is None: - eapi = read_corresponding_eapi_file(myfilename) - # We need to call keys() here in order to avoid the possibility of - # "RuntimeError: dictionary changed size during iteration" - # when an invalid atom is deleted. + if recursive: + file_list = _recursive_file_list(myfilename) + else: + file_list = [myfilename] + atoms = {} - for k, v in pkgs.items(): - try: - k = Atom(k, allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) - except InvalidAtom as e: - writemsg(_("--- Invalid atom in %s: %s\n") % (myfilename, e), - noiselevel=-1) - else: - atoms[k] = v + for filename in file_list: + d = grabdict(filename, juststrings=False, + empty=True, recursive=False, incremental=True) + if not d: + continue + if verify_eapi and eapi is None: + eapi = read_corresponding_eapi_file(myfilename) + + for k, v in d.items(): + try: + k = Atom(k, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) + except InvalidAtom as e: + writemsg(_("--- Invalid atom in %s: %s\n") % (filename, e), + noiselevel=-1) + else: + atoms.setdefault(k, []).extend(v) + + if juststrings: + for k, v in atoms.items(): + atoms[k] = " ".join(v) + return atoms def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=False, allow_repo=False, @@ -447,10 +478,10 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=Fals try: pkg = Atom(pkg, allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: - writemsg(_("--- Invalid atom in %s: %s\n") % (myfilename, e), + writemsg(_("--- Invalid atom in %s: %s\n") % (source_file, e), noiselevel=-1) else: - if pkg_orig == str(pkg): + if pkg_orig == _unicode(pkg): # normal atom, so return as Atom instance if remember_source_file: atoms.append((pkg, source_file)) @@ -464,34 +495,73 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=Fals atoms.append(pkg_orig) return atoms +def _recursive_basename_filter(f): + return not f.startswith(".") and not f.endswith("~") + +def _recursive_file_list(path): + # path may be a regular file or a directory + + def onerror(e): + if e.errno == PermissionDenied.errno: + raise PermissionDenied(path) + + stack = [os.path.split(path)] + + while stack: + parent, fname = stack.pop() + fullpath = os.path.join(parent, fname) + + try: + st = os.stat(fullpath) + except OSError as e: + onerror(e) + continue + + if stat.S_ISDIR(st.st_mode): + if fname in VCS_DIRS or not _recursive_basename_filter(fname): + continue + try: + children = os.listdir(fullpath) + except OSError as e: + onerror(e) + continue + + # Sort in reverse, since we pop from the end of the stack. + # Include regular files in the stack, so files are sorted + # together with directories. + children.sort(reverse=True) + stack.extend((fullpath, x) for x in children) + + elif stat.S_ISREG(st.st_mode): + if _recursive_basename_filter(fname): + yield fullpath + def grablines(myfilename, recursive=0, remember_source_file=False): - mylines=[] - if recursive and os.path.isdir(myfilename): - if os.path.basename(myfilename) in _ignorecvs_dirs: - return mylines - dirlist = os.listdir(myfilename) - dirlist.sort() - for f in dirlist: - if not f.startswith(".") and not f.endswith("~"): - mylines.extend(grablines( - os.path.join(myfilename, f), recursive, remember_source_file)) + mylines = [] + if recursive: + for f in _recursive_file_list(myfilename): + mylines.extend(grablines(f, recursive=False, + remember_source_file=remember_source_file)) + else: try: - myfile = io.open(_unicode_encode(myfilename, + with io.open(_unicode_encode(myfilename, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - if remember_source_file: - mylines = [(line, myfilename) for line in myfile.readlines()] - else: - mylines = myfile.readlines() - myfile.close() + mode='r', encoding=_encodings['content'], errors='replace') as myfile: + if remember_source_file: + mylines = [(line, myfilename) for line in myfile.readlines()] + else: + mylines = myfile.readlines() except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(myfilename) - pass + elif e.errno in (errno.ENOENT, errno.ESTALE): + pass + else: + raise return mylines -def writedict(mydict,myfilename,writekey=True): +def writedict(mydict, myfilename, writekey=True): """Writes out a dict to a file; writekey=0 mode doesn't write out the key and assumes all values are strings, not lists.""" lines = [] @@ -517,18 +587,39 @@ def shlex_split(s): rval = [_unicode_decode(x) for x in rval] return rval -class _tolerant_shlex(shlex.shlex): +class _getconfig_shlex(shlex.shlex): + + def __init__(self, portage_tolerant=False, **kwargs): + shlex.shlex.__init__(self, **kwargs) + self.__portage_tolerant = portage_tolerant + def sourcehook(self, newfile): try: return shlex.shlex.sourcehook(self, newfile) except EnvironmentError as e: - writemsg(_("!!! Parse error in '%s': source command failed: %s\n") % \ - (self.infile, str(e)), noiselevel=-1) + if e.errno == PermissionDenied.errno: + raise PermissionDenied(newfile) + if e.errno not in (errno.ENOENT, errno.ENOTDIR): + writemsg("open('%s', 'r'): %s\n" % (newfile, e), noiselevel=-1) + raise + + msg = self.error_leader() + if e.errno == errno.ENOTDIR: + msg += _("%s: Not a directory") % newfile + else: + msg += _("%s: No such file or directory") % newfile + + if self.__portage_tolerant: + writemsg("%s\n" % msg, noiselevel=-1) + else: + raise ParseError(msg) return (newfile, io.StringIO()) _invalid_var_name_re = re.compile(r'^\d|\W') -def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): +def getconfig(mycfg, tolerant=False, allow_sourcing=False, expand=True, + recursive=False): + if isinstance(expand, dict): # Some existing variable definitions have been # passed in, for use in substitutions. @@ -537,6 +628,21 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: expand_map = {} mykeys = {} + + if recursive: + # Emulate source commands so that syntax error messages + # can display real file names and line numbers. + if not expand: + expand_map = False + fname = None + for fname in _recursive_file_list(mycfg): + mykeys.update(getconfig(fname, tolerant=tolerant, + allow_sourcing=allow_sourcing, expand=expand_map, + recursive=False) or {}) + if fname is None: + return None + return mykeys + f = None try: # NOTE: shlex doesn't support unicode objects with Python 2 @@ -561,49 +667,53 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if f is not None: f.close() + # Since this file has unicode_literals enabled, and Python 2's + # shlex implementation does not support unicode, the following code + # uses _native_string() to encode unicode literals when necessary. + # Workaround for avoiding a silent error in shlex that is # triggered by a source statement at the end of the file # without a trailing newline after the source statement. - if content and content[-1] != '\n': - content += '\n' + if content and content[-1] != portage._native_string('\n'): + content += portage._native_string('\n') # Warn about dos-style line endings since that prevents # people from being able to source them with bash. - if '\r' in content: + if portage._native_string('\r') in content: writemsg(("!!! " + _("Please use dos2unix to convert line endings " + \ "in config file: '%s'") + "\n") % mycfg, noiselevel=-1) lex = None try: - if tolerant: - shlex_class = _tolerant_shlex - else: - shlex_class = shlex.shlex # The default shlex.sourcehook() implementation # only joins relative paths when the infile # attribute is properly set. - lex = shlex_class(content, infile=mycfg, posix=True) - lex.wordchars = string.digits + string.ascii_letters + \ - "~!@#$%*_\:;?,./-+{}" - lex.quotes="\"'" + lex = _getconfig_shlex(instream=content, infile=mycfg, posix=True, + portage_tolerant=tolerant) + lex.wordchars = portage._native_string(string.digits + + string.ascii_letters + "~!@#$%*_\:;?,./-+{}") + lex.quotes = portage._native_string("\"'") if allow_sourcing: - lex.source="source" - while 1: - key=lex.get_token() + lex.source = portage._native_string("source") + + while True: + key = _unicode_decode(lex.get_token()) if key == "export": - key = lex.get_token() + key = _unicode_decode(lex.get_token()) if key is None: #normal end of file - break; - equ=lex.get_token() - if (equ==''): + break + + equ = _unicode_decode(lex.get_token()) + if not equ: msg = lex.error_leader() + _("Unexpected EOF") if not tolerant: raise ParseError(msg) else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - elif (equ!='='): + + elif equ != "=": msg = lex.error_leader() + \ _("Invalid token '%s' (not '=')") % (equ,) if not tolerant: @@ -611,7 +721,8 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - val=lex.get_token() + + val = _unicode_decode(lex.get_token()) if val is None: msg = lex.error_leader() + \ _("Unexpected end of config file: variable '%s'") % (key,) @@ -620,8 +731,6 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - key = _unicode_decode(key) - val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: msg = lex.error_leader() + \ @@ -642,7 +751,7 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): except Exception as e: if isinstance(e, ParseError) or lex is None: raise - msg = _unicode_decode("%s%s") % (lex.error_leader(), e) + msg = "%s%s" % (lex.error_leader(), e) writemsg("%s\n" % msg, noiselevel=-1) raise @@ -660,10 +769,10 @@ def varexpand(mystring, mydict=None, error_leader=None): This code is used by the configfile code, as well as others (parser) This would be a good bunch of code to port to C. """ - numvars=0 - #in single, double quotes - insing=0 - indoub=0 + numvars = 0 + # in single, double quotes + insing = 0 + indoub = 0 pos = 0 length = len(mystring) newstring = [] @@ -675,7 +784,7 @@ def varexpand(mystring, mydict=None, error_leader=None): else: newstring.append("'") # Quote removal is handled by shlex. insing=not insing - pos=pos+1 + pos += 1 continue elif current == '"': if (insing): @@ -683,9 +792,9 @@ def varexpand(mystring, mydict=None, error_leader=None): else: newstring.append('"') # Quote removal is handled by shlex. indoub=not indoub - pos=pos+1 + pos += 1 continue - if (not insing): + if not insing: #expansion time if current == "\n": #convert newlines to spaces @@ -700,7 +809,7 @@ def varexpand(mystring, mydict=None, error_leader=None): # escaped newline characters. Note that we don't handle # escaped quotes here, since getconfig() uses shlex # to handle that earlier. - if (pos+1>=len(mystring)): + if pos + 1 >= len(mystring): newstring.append(current) break else: @@ -722,15 +831,15 @@ def varexpand(mystring, mydict=None, error_leader=None): newstring.append(mystring[pos - 2:pos]) continue elif current == "$": - pos=pos+1 - if mystring[pos]=="{": - pos=pos+1 - braced=True + pos += 1 + if mystring[pos] == "{": + pos += 1 + braced = True else: - braced=False - myvstart=pos + braced = False + myvstart = pos while mystring[pos] in _varexpand_word_chars: - if (pos+1)>=len(mystring): + if pos + 1 >= len(mystring): if braced: msg = _varexpand_unexpected_eof_msg if error_leader is not None: @@ -738,20 +847,20 @@ def varexpand(mystring, mydict=None, error_leader=None): writemsg(msg + "\n", noiselevel=-1) return "" else: - pos=pos+1 + pos += 1 break - pos=pos+1 - myvarname=mystring[myvstart:pos] + pos += 1 + myvarname = mystring[myvstart:pos] if braced: - if mystring[pos]!="}": + if mystring[pos] != "}": msg = _varexpand_unexpected_eof_msg if error_leader is not None: msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" else: - pos=pos+1 - if len(myvarname)==0: + pos += 1 + if len(myvarname) == 0: msg = "$" if braced: msg += "{}" @@ -760,7 +869,7 @@ def varexpand(mystring, mydict=None, error_leader=None): msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" - numvars=numvars+1 + numvars += 1 if myvarname in mydict: newstring.append(mydict[myvarname]) else: @@ -775,9 +884,9 @@ def varexpand(mystring, mydict=None, error_leader=None): # broken and removed, but can still be imported pickle_write = None -def pickle_read(filename,default=None,debug=0): +def pickle_read(filename, default=None, debug=0): if not os.access(filename, os.R_OK): - writemsg(_("pickle_read(): File not readable. '")+filename+"'\n",1) + writemsg(_("pickle_read(): File not readable. '") + filename + "'\n", 1) return default data = None try: @@ -786,12 +895,12 @@ def pickle_read(filename,default=None,debug=0): mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() - del mypickle,myf - writemsg(_("pickle_read(): Loaded pickle. '")+filename+"'\n",1) + del mypickle, myf + writemsg(_("pickle_read(): Loaded pickle. '") + filename + "'\n", 1) except SystemExit as e: raise except Exception as e: - writemsg(_("!!! Failed to load pickle: ")+str(e)+"\n",1) + writemsg(_("!!! Failed to load pickle: ") + str(e) + "\n", 1) data = default return data @@ -819,6 +928,9 @@ class cmp_sort_key(object): list.sort(), making it easier to port code for python-3.0 compatibility. It works by generating key objects which use the given cmp function to implement their __lt__ method. + + Beginning with Python 2.7 and 3.2, equivalent functionality is provided + by functools.cmp_to_key(). """ __slots__ = ("_cmp_func",) @@ -911,6 +1023,10 @@ def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, modified = False + # Since Python 3.4, chown requires int type (no proxies). + uid = int(uid) + gid = int(gid) + if stat_cached is None: try: if follow_links: @@ -1130,7 +1246,7 @@ class atomic_ofstream(ObjectProxy): object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, encoding=_encodings['fs'], errors='strict'), - mode=mode, **kargs)) + mode=mode, **portage._native_kwargs(kargs))) return except IOError as e: if canonical_path == filename: @@ -1212,7 +1328,7 @@ class atomic_ofstream(ObjectProxy): self.close() def __del__(self): - """If the user does not explicitely call close(), it is + """If the user does not explicitly call close(), it is assumed that an error has occurred, so we abort().""" try: f = object.__getattribute__(self, '_file') @@ -1391,9 +1507,9 @@ class LazyItemsDict(UserDict): lazy_item = self.lazy_items.get(k) if lazy_item is not None: if not lazy_item.singleton: - raise TypeError(_unicode_decode("LazyItemsDict " + \ + raise TypeError("LazyItemsDict " + \ "deepcopy is unsafe with lazy items that are " + \ - "not singletons: key=%s value=%s") % (k, lazy_item,)) + "not singletons: key=%s value=%s" % (k, lazy_item,)) UserDict.__setitem__(result, k_copy, deepcopy(self[k], memo)) return result @@ -1565,13 +1681,13 @@ def find_updated_config_files(target_root, config_protect): """ Return a tuple of configuration files that needs to be updated. The tuple contains lists organized like this: - [ protected_dir, file_list ] + [protected_dir, file_list] If the protected config isn't a protected_dir but a procted_file, list is: - [ protected_file, None ] + [protected_file, None] If no configuration files needs to be updated, None is returned """ - os = _os_merge + encoding = _encodings['fs'] if config_protect: # directories with some protect files in them @@ -1603,10 +1719,24 @@ def find_updated_config_files(target_root, config_protect): mycommand = "find '%s' -maxdepth 1 -name '._cfg????_%s'" % \ os.path.split(x.rstrip(os.path.sep)) mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0" - a = subprocess_getstatusoutput(mycommand) - - if a[0] == 0: - files = a[1].split('\0') + cmd = shlex_split(mycommand) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + cmd = [_unicode_encode(arg, encoding=encoding, errors='strict') + for arg in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], encoding=encoding) + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: + files = output.split('\0') # split always produces an empty string as the last element if files and not files[-1]: del files[-1] diff --git a/portage_with_autodep/pym/portage/util/__init__.pyo b/portage_with_autodep/pym/portage/util/__init__.pyo Binary files differindex 941b286..9bab394 100644 --- a/portage_with_autodep/pym/portage/util/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.py b/portage_with_autodep/pym/portage/util/_desktop_entry.py index 7901780..0b49547 100644 --- a/portage_with_autodep/pym/portage/util/_desktop_entry.py +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.py @@ -1,7 +1,8 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import re import subprocess import sys @@ -10,7 +11,9 @@ try: except ImportError: from ConfigParser import Error as ConfigParserError, RawConfigParser +import portage from portage import _encodings, _unicode_encode, _unicode_decode +from portage.util import writemsg def parse_desktop_entry(path): """ @@ -31,45 +34,71 @@ def parse_desktop_entry(path): encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: - read_file(f) + content = f.read() + + # In Python 3.2, read_file does not support bytes in file names + # (see bug #429544), so use StringIO to hide the file name. + read_file(io.StringIO(content)) return parser -_ignored_service_errors = ( - 'error: required key "Name" in group "Desktop Entry" is not present', - 'error: key "Actions" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', - 'error: key "MimeType" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', +_trivial_warnings = re.compile(r' looks redundant with value ') + +_ignored_errors = ( + # Ignore error for emacs.desktop: + # https://bugs.freedesktop.org/show_bug.cgi?id=35844#c6 + 'error: (will be fatal in the future): value "TextEditor" in key "Categories" in group "Desktop Entry" requires another category to be present among the following categories: Utility', + 'warning: key "Encoding" in group "Desktop Entry" is deprecated' +) + +_ShowIn_exemptions = ( + # See bug #480586. + 'contains an unregistered value "Pantheon"', ) def validate_desktop_entry(path): args = ["desktop-file-validate", path] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [_unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output_lines = _unicode_decode(proc.communicate()[0]).splitlines() proc.wait() if output_lines: - try: - desktop_entry = parse_desktop_entry(path) - except ConfigParserError: - pass - else: - if desktop_entry.has_section("Desktop Entry"): - try: - entry_type = desktop_entry.get("Desktop Entry", "Type") - except ConfigParserError: - pass - else: - if entry_type == "Service": - # Filter false errors for Type=Service (bug #414125). - filtered_output = [] - for line in output_lines: - if line[len(path)+2:] in _ignored_service_errors: - continue - filtered_output.append(line) - output_lines = filtered_output + filtered_output = [] + for line in output_lines: + msg = line[len(path)+2:] + # "hint:" output is new in desktop-file-utils-0.21 + if msg.startswith('hint: ') or msg in _ignored_errors: + continue + if 'for key "NotShowIn" in group "Desktop Entry"' in msg or \ + 'for key "OnlyShowIn" in group "Desktop Entry"' in msg: + exempt = False + for s in _ShowIn_exemptions: + if s in msg: + exempt = True + break + if exempt: + continue + filtered_output.append(line) + output_lines = filtered_output + + if output_lines: + output_lines = [line for line in output_lines + if _trivial_warnings.search(line) is None] return output_lines + +if __name__ == "__main__": + for arg in sys.argv[1:]: + for line in validate_desktop_entry(arg): + writemsg(line + "\n", noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.pyo b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo Binary files differindex 7dec17a..c338c3f 100644 --- a/portage_with_autodep/pym/portage/util/_desktop_entry.pyo +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py index e71ac73..3920f94 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -26,7 +26,7 @@ class LinkageMapELF(object): _soname_map_class = slot_dict_class( ("consumers", "providers"), prefix="") - class _obj_properies_class(object): + class _obj_properties_class(object): __slots__ = ("arch", "needed", "runpaths", "soname", "alt_paths", "owner",) @@ -316,7 +316,7 @@ class LinkageMapELF(object): myprops = obj_properties.get(obj_key) if myprops is None: indexed = False - myprops = self._obj_properies_class( + myprops = self._obj_properties_class( arch, needed, path, soname, [], owner) obj_properties[obj_key] = myprops # All object paths are added into the obj_properties tuple. @@ -678,7 +678,7 @@ class LinkageMapELF(object): rValue[soname].add(provider) return rValue - def findConsumers(self, obj, exclude_providers=None): + def findConsumers(self, obj, exclude_providers=None, greedy=True): """ Find consumers of an object or object key. @@ -715,6 +715,9 @@ class LinkageMapELF(object): '/usr/lib/libssl.so.0.9.8'), and return True if the library is owned by a provider which is planned for removal. @type exclude_providers: collection + @param greedy: If True, then include consumers that are satisfied + by alternative providers, otherwise omit them. Default is True. + @type greedy: Boolean @rtype: set of strings (example: set(['/bin/foo', '/usr/bin/bar'])) @return: The return value is a soname -> set-of-library-paths, where set-of-library-paths satisfy soname. @@ -769,16 +772,19 @@ class LinkageMapELF(object): defpath_keys = set(self._path_key(x) for x in self._defpath) satisfied_consumer_keys = set() if soname_node is not None: - if exclude_providers is not None: + if exclude_providers is not None or not greedy: relevant_dir_keys = set() for provider_key in soname_node.providers: + if not greedy and provider_key == obj_key: + continue provider_objs = self._obj_properties[provider_key].alt_paths for p in provider_objs: provider_excluded = False - for excluded_provider_isowner in exclude_providers: - if excluded_provider_isowner(p): - provider_excluded = True - break + if exclude_providers is not None: + for excluded_provider_isowner in exclude_providers: + if excluded_provider_isowner(p): + provider_excluded = True + break if not provider_excluded: # This provider is not excluded. It will # satisfy a consumer of this soname if it diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo Binary files differindex c1e5603..4907057 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo Binary files differindex 8cdd7cb..8d18cdc 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo Binary files differindex 960b66e..86d7f56 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py index bbbce52..9ffcc74 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py @@ -1,20 +1,37 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno -import fcntl import logging import os import select import signal +import sys import time +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + +try: + import threading +except ImportError: + import dummy_threading as threading + from portage.util import writemsg_level from ..SlotObject import SlotObject from .PollConstants import PollConstants from .PollSelectAdapter import PollSelectAdapter class EventLoop(object): + """ + An event loop, intended to be compatible with the GLib event loop. + Call the iteration method in order to execute one iteration of the + loop. The idle_add and timeout_add methods serve as thread-safe + means to interact with the loop's thread. + """ supports_multiprocessing = True @@ -43,7 +60,9 @@ class EventLoop(object): that global_event_loop does not need constructor arguments) @type main: bool """ - self._use_signal = main + self._use_signal = main and fcntl is not None + self._thread_rlock = threading.RLock() + self._thread_condition = threading.Condition(self._thread_rlock) self._poll_event_queue = [] self._poll_event_handlers = {} self._poll_event_handler_ids = {} @@ -52,14 +71,48 @@ class EventLoop(object): self._idle_callbacks = {} self._timeout_handlers = {} self._timeout_interval = None - self._poll_obj = create_poll_instance() - self.IO_ERR = PollConstants.POLLERR - self.IO_HUP = PollConstants.POLLHUP - self.IO_IN = PollConstants.POLLIN - self.IO_NVAL = PollConstants.POLLNVAL - self.IO_OUT = PollConstants.POLLOUT - self.IO_PRI = PollConstants.POLLPRI + self._poll_obj = None + try: + select.epoll + except AttributeError: + pass + else: + try: + epoll_obj = select.epoll() + except IOError: + # This happens with Linux 2.4 kernels: + # IOError: [Errno 38] Function not implemented + pass + else: + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(epoll_obj.fileno(), fcntl.F_SETFD, + fcntl.fcntl(epoll_obj.fileno(), + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._poll_obj = _epoll_adapter(epoll_obj) + self.IO_ERR = select.EPOLLERR + self.IO_HUP = select.EPOLLHUP + self.IO_IN = select.EPOLLIN + self.IO_NVAL = 0 + self.IO_OUT = select.EPOLLOUT + self.IO_PRI = select.EPOLLPRI + + if self._poll_obj is None: + self._poll_obj = create_poll_instance() + self.IO_ERR = PollConstants.POLLERR + self.IO_HUP = PollConstants.POLLHUP + self.IO_IN = PollConstants.POLLIN + self.IO_NVAL = PollConstants.POLLNVAL + self.IO_OUT = PollConstants.POLLOUT + self.IO_PRI = PollConstants.POLLPRI self._child_handlers = {} self._sigchld_read = None @@ -67,6 +120,14 @@ class EventLoop(object): self._sigchld_src_id = None self._pid = os.getpid() + def _new_source_id(self): + """ + Generate a new source id. This method is thread-safe. + """ + with self._thread_rlock: + self._event_handler_id += 1 + return self._event_handler_id + def _poll(self, timeout=None): """ All poll() calls pass through here. The poll events @@ -85,9 +146,11 @@ class EventLoop(object): try: self._poll_event_queue.extend(self._poll_obj.poll(timeout)) break - except select.error as e: + except (IOError, select.error) as e: # Silently handle EINTR, which is normal when we have - # received a signal such as SIGINT. + # received a signal such as SIGINT (epoll objects may + # raise IOError rather than select.error, at least in + # Python 3.2). if not (e.args and e.args[0] == errno.EINTR): writemsg_level("\n!!! select error: %s\n" % (e,), level=logging.ERROR, noiselevel=-1) @@ -101,7 +164,19 @@ class EventLoop(object): def iteration(self, *args): """ - Like glib.MainContext.iteration(), runs a single iteration. + Like glib.MainContext.iteration(), runs a single iteration. In order + to avoid blocking forever when may_block is True (the default), + callers must be careful to ensure that at least one of the following + conditions is met: + 1) An event source or timeout is registered which is guaranteed + to trigger at least on event (a call to an idle function + only counts as an event if it returns a False value which + causes it to stop being called) + 2) Another thread is guaranteed to call one of the thread-safe + methods which notify iteration to stop waiting (such as + idle_add or timeout_add). + These rules ensure that iteration is able to block until an event + arrives, without doing any busy waiting that would waste CPU time. @type may_block: bool @param may_block: if True the call may block waiting for an event (default is True). @@ -120,23 +195,32 @@ class EventLoop(object): event_queue = self._poll_event_queue event_handlers = self._poll_event_handlers events_handled = 0 + timeouts_checked = False if not event_handlers: - if self._run_timeouts(): - events_handled += 1 - if not event_handlers: - if not events_handled and may_block and \ - self._timeout_interval is not None: + with self._thread_condition: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True + if not event_handlers and not events_handled and may_block: # Block so that we don't waste cpu time by looping too # quickly. This makes EventLoop useful for code that needs # to wait for timeout callbacks regardless of whether or # not any IO handlers are currently registered. - try: - self._poll(timeout=self._timeout_interval) - except StopIteration: - pass + timeout = self._get_poll_timeout() + if timeout is None: + wait_timeout = None + else: + wait_timeout = float(timeout) / 1000 + # NOTE: In order to avoid a possible infinite wait when + # wait_timeout is None, the previous _run_timeouts() + # call must have returned False *with* _thread_condition + # acquired. Otherwise, we would risk going to sleep after + # our only notify event has already passed. + self._thread_condition.wait(wait_timeout) if self._run_timeouts(): events_handled += 1 + timeouts_checked = True # If any timeouts have executed, then return immediately, # in order to minimize latency in termination of iteration @@ -147,14 +231,18 @@ class EventLoop(object): if not event_queue: if may_block: - if self._child_handlers: - if self._timeout_interval is None: - timeout = self._sigchld_interval - else: - timeout = min(self._sigchld_interval, - self._timeout_interval) - else: - timeout = self._timeout_interval + timeout = self._get_poll_timeout() + + # Avoid blocking for IO if there are any timeout + # or idle callbacks available to process. + if timeout != 0 and not timeouts_checked: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True + if events_handled: + # Minimize latency for loops controlled + # by timeout or idle callback events. + timeout = 0 else: timeout = 0 @@ -170,17 +258,37 @@ class EventLoop(object): while event_queue: events_handled += 1 f, event = event_queue.pop() - x = event_handlers[f] + try: + x = event_handlers[f] + except KeyError: + # This is known to be triggered by the epoll + # implementation in qemu-user-1.2.2, and appears + # to be harmless (see bug #451326). + continue if not x.callback(f, event, *x.args): self.source_remove(x.source_id) - # Run timeouts last, in order to minimize latency in - # termination of iteration loops that they may control. - if self._run_timeouts(): - events_handled += 1 + if not timeouts_checked: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True return bool(events_handled) + def _get_poll_timeout(self): + + with self._thread_rlock: + if self._child_handlers: + if self._timeout_interval is None: + timeout = self._sigchld_interval + else: + timeout = min(self._sigchld_interval, + self._timeout_interval) + else: + timeout = self._timeout_interval + + return timeout + def child_watch_add(self, pid, callback, data=None): """ Like glib.child_watch_add(), sets callback to be called with the @@ -201,18 +309,29 @@ class EventLoop(object): @rtype: int @return: an integer ID """ - self._event_handler_id += 1 - source_id = self._event_handler_id + source_id = self._new_source_id() self._child_handlers[source_id] = self._child_callback_class( callback=callback, data=data, pid=pid, source_id=source_id) if self._use_signal: if self._sigchld_read is None: self._sigchld_read, self._sigchld_write = os.pipe() + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL, fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK) + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFD, + fcntl.fcntl(self._sigchld_read, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + # The IO watch is dynamically registered and unregistered as # needed, since we don't want to consider it as a valid source # of events when there are no child listeners. It's important @@ -276,22 +395,25 @@ class EventLoop(object): """ Like glib.idle_add(), if callback returns False it is automatically removed from the list of event sources and will - not be called again. + not be called again. This method is thread-safe. @type callback: callable @param callback: a function to call @rtype: int @return: an integer ID """ - self._event_handler_id += 1 - source_id = self._event_handler_id - self._idle_callbacks[source_id] = self._idle_callback_class( - args=args, callback=callback, source_id=source_id) + with self._thread_condition: + source_id = self._new_source_id() + self._idle_callbacks[source_id] = self._idle_callback_class( + args=args, callback=callback, source_id=source_id) + self._thread_condition.notify() return source_id def _run_idle_callbacks(self): + # assumes caller has acquired self._thread_rlock if not self._idle_callbacks: - return + return False + state_change = 0 # Iterate of our local list, since self._idle_callbacks can be # modified during the exection of these callbacks. for x in list(self._idle_callbacks.values()): @@ -304,26 +426,32 @@ class EventLoop(object): x.calling = True try: if not x.callback(*x.args): + state_change += 1 self.source_remove(x.source_id) finally: x.calling = False + return bool(state_change) + def timeout_add(self, interval, function, *args): """ Like glib.timeout_add(), interval argument is the number of milliseconds between calls to your function, and your function should return False to stop being called, or True to continue being called. Any additional positional arguments given here - are passed to your function when it's called. + are passed to your function when it's called. This method is + thread-safe. """ - self._event_handler_id += 1 - source_id = self._event_handler_id - self._timeout_handlers[source_id] = \ - self._timeout_handler_class( - interval=interval, function=function, args=args, - source_id=source_id, timestamp=time.time()) - if self._timeout_interval is None or self._timeout_interval > interval: - self._timeout_interval = interval + with self._thread_condition: + source_id = self._new_source_id() + self._timeout_handlers[source_id] = \ + self._timeout_handler_class( + interval=interval, function=function, args=args, + source_id=source_id, timestamp=time.time()) + if self._timeout_interval is None or \ + self._timeout_interval > interval: + self._timeout_interval = interval + self._thread_condition.notify() return source_id def _run_timeouts(self): @@ -333,37 +461,40 @@ class EventLoop(object): if self._poll_child_processes(): calls += 1 - self._run_idle_callbacks() - - if not self._timeout_handlers: - return bool(calls) + with self._thread_rlock: - ready_timeouts = [] - current_time = time.time() - for x in self._timeout_handlers.values(): - elapsed_seconds = current_time - x.timestamp - # elapsed_seconds < 0 means the system clock has been adjusted - if elapsed_seconds < 0 or \ - (x.interval - 1000 * elapsed_seconds) <= 0: - ready_timeouts.append(x) + if self._run_idle_callbacks(): + calls += 1 - # Iterate of our local list, since self._timeout_handlers can be - # modified during the exection of these callbacks. - for x in ready_timeouts: - if x.source_id not in self._timeout_handlers: - # it got cancelled while executing another timeout - continue - if x.calling: - # don't call it recursively - continue - calls += 1 - x.calling = True - try: - x.timestamp = time.time() - if not x.function(*x.args): - self.source_remove(x.source_id) - finally: - x.calling = False + if not self._timeout_handlers: + return bool(calls) + + ready_timeouts = [] + current_time = time.time() + for x in self._timeout_handlers.values(): + elapsed_seconds = current_time - x.timestamp + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds < 0 or \ + (x.interval - 1000 * elapsed_seconds) <= 0: + ready_timeouts.append(x) + + # Iterate of our local list, since self._timeout_handlers can be + # modified during the exection of these callbacks. + for x in ready_timeouts: + if x.source_id not in self._timeout_handlers: + # it got cancelled while executing another timeout + continue + if x.calling: + # don't call it recursively + continue + calls += 1 + x.calling = True + try: + x.timestamp = time.time() + if not x.function(*x.args): + self.source_remove(x.source_id) + finally: + x.calling = False return bool(calls) @@ -385,8 +516,7 @@ class EventLoop(object): """ if f in self._poll_event_handlers: raise AssertionError("fd %d is already registered" % f) - self._event_handler_id += 1 - source_id = self._event_handler_id + source_id = self._new_source_id() self._poll_event_handler_ids[source_id] = f self._poll_event_handlers[f] = self._io_handler_class( args=args, callback=callback, f=f, source_id=source_id) @@ -406,18 +536,21 @@ class EventLoop(object): self.source_remove(self._sigchld_src_id) self._sigchld_src_id = None return True - idle_callback = self._idle_callbacks.pop(reg_id, None) - if idle_callback is not None: - return True - timeout_handler = self._timeout_handlers.pop(reg_id, None) - if timeout_handler is not None: - if timeout_handler.interval == self._timeout_interval: - if self._timeout_handlers: - self._timeout_interval = \ - min(x.interval for x in self._timeout_handlers.values()) - else: - self._timeout_interval = None - return True + + with self._thread_rlock: + idle_callback = self._idle_callbacks.pop(reg_id, None) + if idle_callback is not None: + return True + timeout_handler = self._timeout_handlers.pop(reg_id, None) + if timeout_handler is not None: + if timeout_handler.interval == self._timeout_interval: + if self._timeout_handlers: + self._timeout_interval = min(x.interval + for x in self._timeout_handlers.values()) + else: + self._timeout_interval = None + return True + f = self._poll_event_handler_ids.pop(reg_id, None) if f is None: return False @@ -467,7 +600,12 @@ def can_poll_device(): return _can_poll_device p = select.poll() - p.register(dev_null.fileno(), PollConstants.POLLIN) + try: + p.register(dev_null.fileno(), PollConstants.POLLIN) + except TypeError: + # Jython: Object 'org.python.core.io.FileIO@f8f175' is not watchable + _can_poll_device = False + return _can_poll_device invalid_request = False for f, event in p.poll(): @@ -488,3 +626,37 @@ def create_poll_instance(): if can_poll_device(): return select.poll() return PollSelectAdapter() + +class _epoll_adapter(object): + """ + Wraps a select.epoll instance in order to make it compatible + with select.poll instances. This is necessary since epoll instances + interpret timeout arguments differently. Note that the file descriptor + that is associated with an epoll instance will close automatically when + it is garbage collected, so it's not necessary to close it explicitly. + """ + __slots__ = ('_epoll_obj',) + + def __init__(self, epoll_obj): + self._epoll_obj = epoll_obj + + def register(self, fd, *args): + self._epoll_obj.register(fd, *args) + + def unregister(self, fd): + self._epoll_obj.unregister(fd) + + def poll(self, *args): + if len(args) > 1: + raise TypeError( + "poll expected at most 2 arguments, got " + \ + repr(1 + len(args))) + timeout = -1 + if args: + timeout = args[0] + if timeout is None or timeout < 0: + timeout = -1 + elif timeout != 0: + timeout = float(timeout) / 1000 + + return self._epoll_obj.poll(timeout) diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo Binary files differindex 6ce2883..948018f 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo Binary files differindex d3453a4..d149863 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo Binary files differindex 6c7c953..876c5b1 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py index 17e63d9..244788c 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py @@ -64,7 +64,7 @@ class PollSelectAdapter(object): if timeout is not None and timeout < 0: timeout = None if timeout is not None: - select_args.append(timeout / 1000) + select_args.append(float(timeout) / 1000) select_events = select.select(*select_args) poll_events = [] diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo Binary files differindex e9ecc51..eac94ca 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo Binary files differindex 69864a6..2b2f9a1 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo Binary files differindex 3d57192..7a7f087 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo diff --git a/portage_with_autodep/pym/portage/util/_pty.pyo b/portage_with_autodep/pym/portage/util/_pty.pyo Binary files differindex 70b5eb0..0fa01cf 100644 --- a/portage_with_autodep/pym/portage/util/_pty.pyo +++ b/portage_with_autodep/pym/portage/util/_pty.pyo diff --git a/portage_with_autodep/pym/portage/util/_urlopen.py b/portage_with_autodep/pym/portage/util/_urlopen.py index 307624b..15f041a 100644 --- a/portage_with_autodep/pym/portage/util/_urlopen.py +++ b/portage_with_autodep/pym/portage/util/_urlopen.py @@ -1,7 +1,11 @@ # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import io import sys +from datetime import datetime +from time import mktime +from email.utils import formatdate, parsedate try: from urllib.request import urlopen as _urlopen @@ -14,29 +18,74 @@ except ImportError: import urllib2 as urllib_request from urllib import splituser as urllib_parse_splituser -def urlopen(url): - try: - return _urlopen(url) - except SystemExit: - raise - except Exception: - if sys.hexversion < 0x3000000: - raise - parse_result = urllib_parse.urlparse(url) - if parse_result.scheme not in ("http", "https") or \ - not parse_result.username: - raise - - return _new_urlopen(url) - -def _new_urlopen(url): - # This is experimental code for bug #413983. +if sys.hexversion >= 0x3000000: + long = int + +# to account for the difference between TIMESTAMP of the index' contents +# and the file-'mtime' +TIMESTAMP_TOLERANCE = 5 + +def urlopen(url, if_modified_since=None): parse_result = urllib_parse.urlparse(url) - netloc = urllib_parse_splituser(parse_result.netloc)[1] - url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) - password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() - if parse_result.username is not None: - password_manager.add_password(None, url, parse_result.username, parse_result.password) - auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager) - opener = urllib_request.build_opener(auth_handler) - return opener.open(url) + if parse_result.scheme not in ("http", "https"): + return _urlopen(url) + else: + netloc = urllib_parse_splituser(parse_result.netloc)[1] + url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) + password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() + request = urllib_request.Request(url) + request.add_header('User-Agent', 'Gentoo Portage') + if if_modified_since: + request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) + if parse_result.username is not None: + password_manager.add_password(None, url, parse_result.username, parse_result.password) + auth_handler = CompressedResponseProcessor(password_manager) + opener = urllib_request.build_opener(auth_handler) + hdl = opener.open(request) + if hdl.headers.get('last-modified', ''): + try: + add_header = hdl.headers.add_header + except AttributeError: + # Python 2 + add_header = hdl.headers.addheader + add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified'))) + return hdl + +def _timestamp_to_http(timestamp): + dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE)) + stamp = mktime(dt.timetuple()) + return formatdate(timeval=stamp, localtime=False, usegmt=True) + +def _http_to_timestamp(http_datetime_string): + tuple = parsedate(http_datetime_string) + timestamp = mktime(tuple) + return str(long(timestamp)) + +class CompressedResponseProcessor(urllib_request.HTTPBasicAuthHandler): + # Handler for compressed responses. + + def http_request(self, req): + req.add_header('Accept-Encoding', 'bzip2,gzip,deflate') + return req + https_request = http_request + + def http_response(self, req, response): + decompressed = None + if response.headers.get('content-encoding') == 'bzip2': + import bz2 + decompressed = io.BytesIO(bz2.decompress(response.read())) + elif response.headers.get('content-encoding') == 'gzip': + from gzip import GzipFile + decompressed = GzipFile(fileobj=io.BytesIO(response.read()), mode='r') + elif response.headers.get('content-encoding') == 'deflate': + import zlib + try: + decompressed = io.BytesIO(zlib.decompress(response.read())) + except zlib.error: # they ignored RFC1950 + decompressed = io.BytesIO(zlib.decompress(response.read(), -zlib.MAX_WBITS)) + if decompressed: + old_response = response + response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code) + response.msg = old_response.msg + return response + https_response = http_response diff --git a/portage_with_autodep/pym/portage/util/_urlopen.pyo b/portage_with_autodep/pym/portage/util/_urlopen.pyo Binary files differindex 9f51de8..d548069 100644 --- a/portage_with_autodep/pym/portage/util/_urlopen.pyo +++ b/portage_with_autodep/pym/portage/util/_urlopen.pyo diff --git a/portage_with_autodep/pym/portage/util/digraph.py b/portage_with_autodep/pym/portage/util/digraph.py index f3ae658..fc1fb86 100644 --- a/portage_with_autodep/pym/portage/util/digraph.py +++ b/portage_with_autodep/pym/portage/util/digraph.py @@ -1,12 +1,13 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['digraph'] from collections import deque import sys -from portage import _unicode_decode from portage.util import writemsg class digraph(object): @@ -16,24 +17,24 @@ class digraph(object): def __init__(self): """Create an empty digraph""" - + # { node : ( { child : priority } , { parent : priority } ) } self.nodes = {} self.order = [] def add(self, node, parent, priority=0): """Adds the specified node with the specified parent. - + If the dep is a soft-dep and the node already has a hard relationship to the parent, the relationship is left as hard.""" - + if node not in self.nodes: self.nodes[node] = ({}, {}, node) self.order.append(node) - + if not parent: return - + if parent not in self.nodes: self.nodes[parent] = ({}, {}, parent) self.order.append(parent) @@ -50,15 +51,15 @@ class digraph(object): """Removes the specified node from the digraph, also removing and ties to other nodes in the digraph. Raises KeyError if the node doesn't exist.""" - + if node not in self.nodes: raise KeyError(node) - + for parent in self.nodes[node][1]: del self.nodes[parent][0][node] for child in self.nodes[node][0]: del self.nodes[child][1][node] - + del self.nodes[node] self.order.remove(node) @@ -157,10 +158,10 @@ class digraph(object): def leaf_nodes(self, ignore_priority=None): """Return all nodes that have no children - + If ignore_soft_deps is True, soft deps are not counted as children in calculations.""" - + leaf_nodes = [] if ignore_priority is None: for node in self.order: @@ -191,10 +192,10 @@ class digraph(object): def root_nodes(self, ignore_priority=None): """Return all nodes that have no parents. - + If ignore_soft_deps is True, soft deps are not counted as parents in calculations.""" - + root_nodes = [] if ignore_priority is None: for node in self.order: @@ -272,18 +273,17 @@ class digraph(object): def debug_print(self): def output(s): writemsg(s, noiselevel=-1) - # Use _unicode_decode() to force unicode format + # Use unicode_literals to force unicode format # strings for python-2.x safety, ensuring that # node.__unicode__() is used when necessary. for node in self.nodes: - output(_unicode_decode("%s ") % (node,)) + output("%s " % (node,)) if self.nodes[node][0]: output("depends on\n") else: output("(no children)\n") for child, priorities in self.nodes[node][0].items(): - output(_unicode_decode(" %s (%s)\n") % \ - (child, priorities[-1],)) + output(" %s (%s)\n" % (child, priorities[-1],)) def bfs(self, start, ignore_priority=None): if start not in self: diff --git a/portage_with_autodep/pym/portage/util/digraph.pyo b/portage_with_autodep/pym/portage/util/digraph.pyo Binary files differindex 8e503a6..fc00aa7 100644 --- a/portage_with_autodep/pym/portage/util/digraph.pyo +++ b/portage_with_autodep/pym/portage/util/digraph.pyo diff --git a/portage_with_autodep/pym/portage/util/env_update.py b/portage_with_autodep/pym/portage/util/env_update.py index ace4077..5fddaac 100644 --- a/portage_with_autodep/pym/portage/util/env_update.py +++ b/portage_with_autodep/pym/portage/util/env_update.py @@ -1,16 +1,17 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['env_update'] import errno +import glob import io import stat import sys import time import portage -from portage import os, _encodings, _unicode_encode +from portage import os, _encodings, _unicode_decode, _unicode_encode from portage.checksum import prelink_capable from portage.data import ostype from portage.exception import ParseError @@ -88,7 +89,8 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, eprefix = settings.get("EPREFIX", "") eprefix_lstrip = eprefix.lstrip(os.sep) - envd_dir = os.path.join(target_root, eprefix_lstrip, "etc", "env.d") + eroot = normalize_path(os.path.join(target_root, eprefix_lstrip)).rstrip(os.sep) + os.sep + envd_dir = os.path.join(eroot, "etc", "env.d") ensure_dirs(envd_dir, mode=0o755) fns = listdir(envd_dir, EmptyOnError=1) fns.sort() @@ -164,15 +166,14 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, they won't be overwritten by this dict.update call.""" env.update(myconfig) - ldsoconf_path = os.path.join( - target_root, eprefix_lstrip, "etc", "ld.so.conf") + ldsoconf_path = os.path.join(eroot, "etc", "ld.so.conf") try: myld = io.open(_unicode_encode(ldsoconf_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') - myldlines=myld.readlines() + myldlines = myld.readlines() myld.close() - oldld=[] + oldld = [] for x in myldlines: #each line has at least one char (a newline) if x[:1] == "#": @@ -193,20 +194,34 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, myfd.write(x + "\n") myfd.close() + potential_lib_dirs = set() + for lib_dir_glob in ('usr/lib*', 'lib*'): + x = os.path.join(eroot, lib_dir_glob) + for y in glob.glob(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict')): + try: + y = _unicode_decode(y, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue + if os.path.basename(y) != 'libexec': + potential_lib_dirs.add(y[len(eroot):]) + # Update prelink.conf if we are prelink-enabled if prelink_capable: - newprelink = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "prelink.conf")) + prelink_d = os.path.join(eroot, 'etc', 'prelink.conf.d') + ensure_dirs(prelink_d) + newprelink = atomic_ofstream(os.path.join(prelink_d, 'portage.conf')) newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") newprelink.write("# contents of /etc/env.d directory\n") - for x in ["/bin","/sbin","/usr/bin","/usr/sbin","/lib","/usr/lib"]: - newprelink.write("-l %s\n" % (x,)); - prelink_paths = [] - prelink_paths += specials.get("LDPATH", []) - prelink_paths += specials.get("PATH", []) - prelink_paths += specials.get("PRELINK_PATH", []) - prelink_path_mask = specials.get("PRELINK_PATH_MASK", []) + for x in sorted(potential_lib_dirs) + ['bin', 'sbin']: + newprelink.write('-l /%s\n' % (x,)); + prelink_paths = set() + prelink_paths |= set(specials.get('LDPATH', [])) + prelink_paths |= set(specials.get('PATH', [])) + prelink_paths |= set(specials.get('PRELINK_PATH', [])) + prelink_path_mask = specials.get('PRELINK_PATH_MASK', []) for x in prelink_paths: if not x: continue @@ -227,12 +242,26 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, newprelink.write("-b %s\n" % (x,)) newprelink.close() + # Migration code path. If /etc/prelink.conf was generated by us, then + # point it to the new stuff until the prelink package re-installs. + prelink_conf = os.path.join(eroot, 'etc', 'prelink.conf') + try: + with open(_unicode_encode(prelink_conf, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + if f.readline() == b'# prelink.conf autogenerated by env-update; make all changes to\n': + f = atomic_ofstream(prelink_conf) + f.write('-c /etc/prelink.conf.d/*.conf\n') + f.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + current_time = long(time.time()) mtime_changed = False + lib_dirs = set() - for lib_dir in set(specials["LDPATH"] + \ - ['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): - x = os.path.join(target_root, eprefix_lstrip, lib_dir.lstrip(os.sep)) + for lib_dir in set(specials['LDPATH']) | potential_lib_dirs: + x = os.path.join(eroot, lib_dir.lstrip(os.sep)) try: newldpathtime = os.stat(x)[stat.ST_MTIME] lib_dirs.add(normalize_path(x)) @@ -292,7 +321,7 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \ (target_root,)) os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root)) - elif ostype in ("FreeBSD","DragonFly"): + elif ostype in ("FreeBSD", "DragonFly"): writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \ target_root) os.system(("cd / ; %s -elf -i " + \ @@ -308,11 +337,10 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n" #create /etc/profile.env for bash support - outfile = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "profile.env")) + outfile = atomic_ofstream(os.path.join(eroot, "etc", "profile.env")) outfile.write(penvnotice) - env_keys = [ x for x in env if x != "LDPATH" ] + env_keys = [x for x in env if x != "LDPATH"] env_keys.sort() for k in env_keys: v = env[k] @@ -323,8 +351,7 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, outfile.close() #create /etc/csh.env for (t)csh support - outfile = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "csh.env")) + outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env")) outfile.write(cenvnotice) for x in env_keys: outfile.write("setenv %s '%s'\n" % (x, env[x])) diff --git a/portage_with_autodep/pym/portage/util/env_update.pyo b/portage_with_autodep/pym/portage/util/env_update.pyo Binary files differindex ee3b187..2398289 100644 --- a/portage_with_autodep/pym/portage/util/env_update.pyo +++ b/portage_with_autodep/pym/portage/util/env_update.pyo diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.py b/portage_with_autodep/pym/portage/util/lafilefixer.py index 54ff20d..2562d9a 100644 --- a/portage_with_autodep/pym/portage/util/lafilefixer.py +++ b/portage_with_autodep/pym/portage/util/lafilefixer.py @@ -11,7 +11,7 @@ from portage.exception import InvalidData # This an re-implementaion of dev-util/lafilefixer-0.5. # rewrite_lafile() takes the contents of an lafile as a string # It then parses the dependency_libs and inherited_linker_flags -# entries. +# entries. # We insist on dependency_libs being present. inherited_linker_flags # is optional. # There are strict rules about the syntax imposed by libtool's libltdl. @@ -21,7 +21,7 @@ from portage.exception import InvalidData # lafilefixer does). # What it does: # * Replaces all .la files with absolut paths in dependency_libs with -# corresponding -l* and -L* entries +# corresponding -l* and -L* entries # (/usr/lib64/libfoo.la -> -L/usr/lib64 -lfoo) # * Moves various flags (see flag_re below) to inherited_linker_flags, # if such an entry was present. @@ -36,7 +36,7 @@ from portage.exception import InvalidData dep_libs_re = re.compile(b"dependency_libs='(?P<value>[^']*)'$") inh_link_flags_re = re.compile(b"inherited_linker_flags='(?P<value>[^']*)'$") -#regexes for replacing stuff in -L entries. +#regexes for replacing stuff in -L entries. #replace 'X11R6/lib' and 'local/lib' with 'lib', no idea what's this about. X11_local_sub = re.compile(b"X11R6/lib|local/lib") #get rid of the '..' @@ -129,11 +129,11 @@ def rewrite_lafile(contents): #This allows us to place all -L entries at the beginning #of 'dependency_libs'. ladir = dep_libs_entry - + ladir = X11_local_sub.sub(b"lib", ladir) ladir = pkgconfig_sub1.sub(b"usr", ladir) ladir = pkgconfig_sub2.sub(b"\g<usrlib>", ladir) - + if ladir not in libladir: libladir.append(ladir) diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.pyo b/portage_with_autodep/pym/portage/util/lafilefixer.pyo Binary files differindex a6e06ab..5378d82 100644 --- a/portage_with_autodep/pym/portage/util/lafilefixer.pyo +++ b/portage_with_autodep/pym/portage/util/lafilefixer.pyo diff --git a/portage_with_autodep/pym/portage/util/listdir.py b/portage_with_autodep/pym/portage/util/listdir.py index c2628cb..2012e14 100644 --- a/portage_with_autodep/pym/portage/util/listdir.py +++ b/portage_with_autodep/pym/portage/util/listdir.py @@ -1,36 +1,33 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['cacheddir', 'listdir'] import errno import stat -import time +import sys + +if sys.hexversion < 0x3000000: + from itertools import izip as zip from portage import os +from portage.const import VCS_DIRS from portage.exception import DirectoryNotFound, PermissionDenied, PortageException -from portage.util import normalize_path, writemsg - -_ignorecvs_dirs = ('CVS', 'RCS', 'SCCS', '.svn', '.git') +from portage.util import normalize_path + +# The global dircache is no longer supported, since it could +# be a memory leak for API consumers. Any cacheddir callers +# should use higher-level caches instead, when necessary. +# TODO: Remove dircache variable after stable portage does +# not use is (keep it for now, in case API consumers clear +# it manually). dircache = {} -cacheHit = 0 -cacheMiss = 0 -cacheStale = 0 def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymlinks=True): - global cacheHit,cacheMiss,cacheStale mypath = normalize_path(my_original_path) - if mypath in dircache: - cacheHit += 1 - cached_mtime, list, ftype = dircache[mypath] - else: - cacheMiss += 1 - cached_mtime, list, ftype = -1, [], [] try: pathstat = os.stat(mypath) - if stat.S_ISDIR(pathstat[stat.ST_MODE]): - mtime = pathstat.st_mtime - else: + if not stat.S_ISDIR(pathstat.st_mode): raise DirectoryNotFound(mypath) except EnvironmentError as e: if e.errno == PermissionDenied.errno: @@ -39,19 +36,16 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli return [], [] except PortageException: return [], [] - # Python retuns mtime in seconds, so if it was changed in the last few seconds, it could be invalid - if mtime != cached_mtime or time.time() - mtime < 4: - if mypath in dircache: - cacheStale += 1 + else: try: - list = os.listdir(mypath) + fpaths = os.listdir(mypath) except EnvironmentError as e: if e.errno != errno.EACCES: raise del e raise PermissionDenied(mypath) ftype = [] - for x in list: + for x in fpaths: try: if followSymlinks: pathstat = os.stat(mypath+"/"+x) @@ -68,23 +62,22 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli ftype.append(3) except (IOError, OSError): ftype.append(3) - dircache[mypath] = mtime, list, ftype - - ret_list = [] - ret_ftype = [] - for x in range(0, len(list)): - if list[x] in ignorelist: - pass - elif ignorecvs: - if list[x][:2] != ".#" and \ - not (ftype[x] == 1 and list[x] in _ignorecvs_dirs): - ret_list.append(list[x]) - ret_ftype.append(ftype[x]) - else: - ret_list.append(list[x]) - ret_ftype.append(ftype[x]) - - writemsg("cacheddirStats: H:%d/M:%d/S:%d\n" % (cacheHit, cacheMiss, cacheStale),10) + + if ignorelist or ignorecvs: + ret_list = [] + ret_ftype = [] + for file_path, file_type in zip(fpaths, ftype): + if file_path in ignorelist: + pass + elif ignorecvs: + if file_path[:2] != ".#" and \ + not (file_type == 1 and file_path in VCS_DIRS): + ret_list.append(file_path) + ret_ftype.append(file_type) + else: + ret_list = fpaths + ret_ftype = ftype + return ret_list, ret_ftype def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelist=[], followSymlinks=True, @@ -98,7 +91,7 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis @type recursive: Boolean @param filesonly; Only return files, not more directories @type filesonly: Boolean - @param ignorecvs: Ignore CVS directories ('CVS','SCCS','.svn','.git') + @param ignorecvs: Ignore VCS directories @type ignorecvs: Boolean @param ignorelist: List of filenames/directories to exclude @type ignorelist: List @@ -112,40 +105,35 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis @return: A list of files and directories (or just files or just directories) or an empty list. """ - list, ftype = cacheddir(mypath, ignorecvs, ignorelist, EmptyOnError, followSymlinks) + fpaths, ftype = cacheddir(mypath, ignorecvs, ignorelist, EmptyOnError, followSymlinks) - if list is None: - list=[] + if fpaths is None: + fpaths = [] if ftype is None: - ftype=[] + ftype = [] if not (filesonly or dirsonly or recursive): - return list + return fpaths if recursive: - x=0 - while x<len(ftype): - if ftype[x] == 1: - l,f = cacheddir(mypath+"/"+list[x], ignorecvs, ignorelist, EmptyOnError, - followSymlinks) - - l=l[:] - for y in range(0,len(l)): - l[y]=list[x]+"/"+l[y] - list=list+l - ftype=ftype+f - x+=1 + stack = list(zip(fpaths, ftype)) + fpaths = [] + ftype = [] + while stack: + file_path, file_type = stack.pop() + fpaths.append(file_path) + ftype.append(file_type) + if file_type == 1: + subdir_list, subdir_types = cacheddir( + os.path.join(mypath, file_path), ignorecvs, + ignorelist, EmptyOnError, followSymlinks) + stack.extend((os.path.join(file_path, x), x_type) + for x, x_type in zip(subdir_list, subdir_types)) + if filesonly: - rlist=[] - for x in range(0,len(ftype)): - if ftype[x]==0: - rlist=rlist+[list[x]] + fpaths = [x for x, x_type in zip(fpaths, ftype) if x_type == 0] + elif dirsonly: - rlist = [] - for x in range(0, len(ftype)): - if ftype[x] == 1: - rlist = rlist + [list[x]] - else: - rlist=list + fpaths = [x for x, x_type in zip(fpaths, ftype) if x_type == 1] - return rlist + return fpaths diff --git a/portage_with_autodep/pym/portage/util/listdir.pyo b/portage_with_autodep/pym/portage/util/listdir.pyo Binary files differindex 0f02d6d..e4d40be 100644 --- a/portage_with_autodep/pym/portage/util/listdir.pyo +++ b/portage_with_autodep/pym/portage/util/listdir.pyo diff --git a/portage_with_autodep/pym/portage/util/movefile.py b/portage_with_autodep/pym/portage/util/movefile.py index 10577b5..452e77f 100644 --- a/portage_with_autodep/pym/portage/util/movefile.py +++ b/portage_with_autodep/pym/portage/util/movefile.py @@ -1,18 +1,22 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['movefile'] import errno +import fnmatch import os as _os import shutil as _shutil import stat +import sys import subprocess import textwrap import portage from portage import bsd_chflags, _encodings, _os_overrides, _selinux, \ - _unicode_decode, _unicode_encode, _unicode_func_wrapper,\ + _unicode_decode, _unicode_encode, _unicode_func_wrapper, \ _unicode_module_wrapper from portage.const import MOVE_BINARY from portage.exception import OperationNotSupported @@ -24,43 +28,113 @@ def _apply_stat(src_stat, dest): _os.chown(dest, src_stat.st_uid, src_stat.st_gid) _os.chmod(dest, stat.S_IMODE(src_stat.st_mode)) +_xattr_excluder_cache = {} + +def _get_xattr_excluder(pattern): + + try: + value = _xattr_excluder_cache[pattern] + except KeyError: + value = _xattr_excluder(pattern) + _xattr_excluder_cache[pattern] = value + + return value + +class _xattr_excluder(object): + + __slots__ = ('_pattern_split',) + + def __init__(self, pattern): + + if pattern is None: + self._pattern_split = None + else: + pattern = pattern.split() + if not pattern: + self._pattern_split = None + else: + pattern.sort() + self._pattern_split = tuple(pattern) + + def __call__(self, attr): + + if self._pattern_split is None: + return False + + match = fnmatch.fnmatch + for x in self._pattern_split: + if match(attr, x): + return True + + return False + if hasattr(_os, "getxattr"): # Python >=3.3 and GNU/Linux - def _copyxattr(src, dest): - for attr in _os.listxattr(src): + def _copyxattr(src, dest, exclude=None): + + try: + attrs = _os.listxattr(src) + except OSError as e: + if e.errno != OperationNotSupported.errno: + raise + attrs = () + if attrs: + if exclude is not None and isinstance(attrs[0], bytes): + exclude = exclude.encode(_encodings['fs']) + exclude = _get_xattr_excluder(exclude) + + for attr in attrs: + if exclude(attr): + continue try: _os.setxattr(dest, attr, _os.getxattr(src, attr)) raise_exception = False except OSError: raise_exception = True if raise_exception: - raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + raise OperationNotSupported(_("Filesystem containing file '%s' " + "does not support extended attribute '%s'") % + (_unicode_decode(dest), _unicode_decode(attr))) else: try: import xattr except ImportError: xattr = None if xattr is not None: - def _copyxattr(src, dest): - for attr in xattr.list(src): + def _copyxattr(src, dest, exclude=None): + + try: + attrs = xattr.list(src) + except IOError as e: + if e.errno != OperationNotSupported.errno: + raise + attrs = () + + if attrs: + if exclude is not None and isinstance(attrs[0], bytes): + exclude = exclude.encode(_encodings['fs']) + exclude = _get_xattr_excluder(exclude) + + for attr in attrs: + if exclude(attr): + continue try: xattr.set(dest, attr, xattr.get(src, attr)) raise_exception = False except IOError: raise_exception = True if raise_exception: - raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + raise OperationNotSupported(_("Filesystem containing file '%s' " + "does not support extended attribute '%s'") % + (_unicode_decode(dest), _unicode_decode(attr))) else: - _devnull = open("/dev/null", "wb") try: - subprocess.call(["getfattr", "--version"], stdout=_devnull) - subprocess.call(["setfattr", "--version"], stdout=_devnull) - _has_getfattr_and_setfattr = True + with open(_os.devnull, 'wb') as f: + subprocess.call(["getfattr", "--version"], stdout=f) + subprocess.call(["setfattr", "--version"], stdout=f) except OSError: - _has_getfattr_and_setfattr = False - _devnull.close() - if _has_getfattr_and_setfattr: - def _copyxattr(src, dest): + def _copyxattr(src, dest, exclude=None): + # TODO: implement exclude getfattr_process = subprocess.Popen(["getfattr", "-d", "--absolute-names", src], stdout=subprocess.PIPE) getfattr_process.wait() extended_attributes = getfattr_process.stdout.readlines() @@ -72,14 +146,15 @@ else: if setfattr_process.returncode != 0: raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) else: - def _copyxattr(src, dest): + def _copyxattr(src, dest, exclude=None): pass def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, hardlink_candidates=None, encoding=_encodings['fs']): """moves a file from src to dest, preserving all permissions and attributes; mtime will - be preserved even when moving across filesystems. Returns true on success and false on - failure. Move is atomic.""" + be preserved even when moving across filesystems. Returns mtime as integer on success + and None on failure. mtime is expressed in seconds in Python <3.3 and nanoseconds in + Python >=3.3. Move is atomic.""" if mysettings is None: mysettings = portage.settings @@ -102,22 +177,22 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, try: if not sstat: - sstat=os.lstat(src) + sstat = os.lstat(src) except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("Stating source file failed... movefile()"), noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None - destexists=1 + destexists = 1 try: - dstat=os.lstat(dest) + dstat = os.lstat(dest) except (OSError, IOError): - dstat=os.lstat(os.path.dirname(dest)) - destexists=0 + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 if bsd_chflags: if destexists and dstat.st_flags != 0: @@ -132,7 +207,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) - destexists=0 + destexists = 0 except SystemExit as e: raise except Exception as e: @@ -140,7 +215,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(sstat[stat.ST_MODE]): try: - target=os.readlink(src) + target = os.readlink(src) if mysettings and "D" in mysettings and \ target.startswith(mysettings["D"]): target = target[len(mysettings["D"])-1:] @@ -159,17 +234,32 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if e.errno not in (errno.ENOENT, errno.EEXIST) or \ target != os.readlink(dest): raise - lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) - # utime() only works on the target of a symlink, so it's not - # possible to perserve mtime on symlinks. - return os.lstat(dest)[stat.ST_MTIME] + lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) + + try: + _os.unlink(src_bytes) + except OSError: + pass + + if sys.hexversion >= 0x3030000: + try: + os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False) + except NotImplementedError: + # utimensat() and lutimes() missing in libc. + return os.stat(dest, follow_symlinks=False).st_mtime_ns + else: + return sstat.st_mtime_ns + else: + # utime() in Python <3.3 only works on the target of a symlink, so it's not + # possible to preserve mtime on symlinks. + return os.lstat(dest)[stat.ST_MTIME] except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("failed to properly create symlink:"), noiselevel=-1) writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None hardlinked = False @@ -204,9 +294,13 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, writemsg("!!! %s\n" % (e,), noiselevel=-1) return None hardlinked = True + try: + _os.unlink(src_bytes) + except OSError: + pass break - renamefailed=1 + renamefailed = 1 if hardlinked: renamefailed = False if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev): @@ -214,14 +308,14 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if selinux_enabled: selinux.rename(src, dest) else: - os.rename(src,dest) - renamefailed=0 + os.rename(src, dest) + renamefailed = 0 except OSError as e: if e.errno != errno.EXDEV: # Some random error. writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") % {"src": src, "dest": dest}, noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None # Invalid cross-device-link 'bind' mounted or actually Cross-Device if renamefailed: @@ -233,7 +327,8 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, _copyfile(src_bytes, dest_tmp_bytes) if xattr_enabled: try: - _copyxattr(src_bytes, dest_tmp_bytes) + _copyxattr(src_bytes, dest_tmp_bytes, + exclude=mysettings.get("PORTAGE_XATTR_EXCLUDE", "security.* system.nfs4_acl")) except SystemExit: raise except: @@ -252,7 +347,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except Exception as e: writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % {"src": src, "dest": dest}, noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None else: #we don't yet handle special, so we need to fall back to /bin/mv @@ -265,35 +360,54 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, writemsg("!!! %s\n" % a, noiselevel=-1) return None # failure - # Always use stat_obj[stat.ST_MTIME] for the integral timestamp which - # is returned, since the stat_obj.st_mtime float attribute rounds *up* + # In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp + # which is returned, since the stat_obj.st_mtime float attribute rounds *up* # if the nanosecond part of the timestamp is 999999881 ns or greater. try: if hardlinked: - newmtime = os.stat(dest)[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = os.stat(dest).st_mtime_ns + else: + newmtime = os.stat(dest)[stat.ST_MTIME] else: # Note: It is not possible to preserve nanosecond precision # (supported in POSIX.1-2008 via utimensat) with the IEEE 754 # double precision float which only has a 53 bit significand. if newmtime is not None: - os.utime(dest, (newmtime, newmtime)) + if sys.hexversion >= 0x3030000: + os.utime(dest, ns=(newmtime, newmtime)) + else: + os.utime(dest, (newmtime, newmtime)) else: - newmtime = sstat[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = sstat.st_mtime_ns + else: + newmtime = sstat[stat.ST_MTIME] if renamefailed: - # If rename succeeded then timestamps are automatically - # preserved with complete precision because the source - # and destination inode are the same. Otherwise, round - # down to the nearest whole second since python's float - # st_mtime cannot be used to preserve the st_mtim.tv_nsec - # field with complete precision. Note that we have to use - # stat_obj[stat.ST_MTIME] here because the float - # stat_obj.st_mtime rounds *up* sometimes. - os.utime(dest, (newmtime, newmtime)) + if sys.hexversion >= 0x3030000: + # If rename succeeded then timestamps are automatically + # preserved with complete precision because the source + # and destination inodes are the same. Otherwise, manually + # update timestamps with nanosecond precision. + os.utime(dest, ns=(newmtime, newmtime)) + else: + # If rename succeeded then timestamps are automatically + # preserved with complete precision because the source + # and destination inodes are the same. Otherwise, round + # down to the nearest whole second since python's float + # st_mtime cannot be used to preserve the st_mtim.tv_nsec + # field with complete precision. Note that we have to use + # stat_obj[stat.ST_MTIME] here because the float + # stat_obj.st_mtime rounds *up* sometimes. + os.utime(dest, (newmtime, newmtime)) except OSError: # The utime can fail here with EPERM even though the move succeeded. # Instead of failing, use stat to return the mtime if possible. try: - newmtime = os.stat(dest)[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = os.stat(dest).st_mtime_ns + else: + newmtime = os.stat(dest)[stat.ST_MTIME] except OSError as e: writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1) writemsg("!!! %s\n" % dest, noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/util/movefile.pyo b/portage_with_autodep/pym/portage/util/movefile.pyo Binary files differindex 1228ee7..690b9b6 100644 --- a/portage_with_autodep/pym/portage/util/movefile.pyo +++ b/portage_with_autodep/pym/portage/util/movefile.pyo diff --git a/portage_with_autodep/pym/portage/util/mtimedb.pyo b/portage_with_autodep/pym/portage/util/mtimedb.pyo Binary files differindex fda479a..6fe9b37 100644 --- a/portage_with_autodep/pym/portage/util/mtimedb.pyo +++ b/portage_with_autodep/pym/portage/util/mtimedb.pyo diff --git a/portage_with_autodep/pym/portage/util/whirlpool.py b/portage_with_autodep/pym/portage/util/whirlpool.py index c696f6f..170ae73 100644 --- a/portage_with_autodep/pym/portage/util/whirlpool.py +++ b/portage_with_autodep/pym/portage/util/whirlpool.py @@ -639,6 +639,8 @@ def WhirlpoolInit(ctx): return def WhirlpoolAdd(source, sourceBits, ctx): + if not isinstance(source, bytes): + raise TypeError("Expected %s, got %s" % (bytes, type(source))) if sys.hexversion < 0x3000000: source = [ord(s)&0xff for s in source] diff --git a/portage_with_autodep/pym/portage/util/whirlpool.pyo b/portage_with_autodep/pym/portage/util/whirlpool.pyo Binary files differindex 4bcf49a..98e2815 100644 --- a/portage_with_autodep/pym/portage/util/whirlpool.pyo +++ b/portage_with_autodep/pym/portage/util/whirlpool.pyo diff --git a/portage_with_autodep/pym/portage/versions.py b/portage_with_autodep/pym/portage/versions.py index db14e99..3bfc388 100644 --- a/portage_with_autodep/pym/portage/versions.py +++ b/portage_with_autodep/pym/portage/versions.py @@ -1,7 +1,9 @@ # versions.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'best', 'catpkgsplit', 'catsplit', 'cpv_getkey', 'cpv_getversion', 'cpv_sort_key', 'pkgcmp', 'pkgsplit', @@ -23,7 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:cmp_sort_key', ) from portage import _unicode_decode -from portage.eapi import eapi_allows_dots_in_PN +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidData from portage.localization import _ @@ -31,6 +33,10 @@ _unknown_repo = "__unknown__" # \w is [a-zA-Z0-9_] +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot = r'([\w+][\w+.-]*)' + # 2.1.1 A category name may contain any of the characters [A-Za-z0-9+_.-]. # It must not begin with a hyphen or a dot. _cat = r'[\w+][\w+.-]*' @@ -65,6 +71,42 @@ suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1} endversion_keys = ["pre", "p", "alpha", "beta", "rc"] +_slot_re_cache = {} + +def _get_slot_re(eapi_attrs): + cache_key = eapi_attrs.slot_operator + slot_re = _slot_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_operator: + slot_re = _slot + r'(/' + _slot + r')?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE | re.UNICODE) + + _slot_re_cache[cache_key] = slot_re + return slot_re + +_pv_re_cache = {} + +def _get_pv_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + pv_re = _pv_re_cache.get(cache_key) + if pv_re is not None: + return pv_re + + if eapi_attrs.dots_in_PN: + pv_re = _pv['dots_allowed_in_PN'] + else: + pv_re = _pv['dots_disallowed_in_PN'] + + pv_re = re.compile(r'^' + pv_re + r'$', re.VERBOSE | re.UNICODE) + + _pv_re_cache[cache_key] = pv_re + return pv_re + def ververify(myver, silent=1): if ver_regexp.match(myver): return 1 @@ -251,17 +293,6 @@ def pkgcmp(pkg1, pkg2): return None return vercmp("-".join(pkg1[1:]), "-".join(pkg2[1:])) -_pv_re = { - "dots_disallowed_in_PN": re.compile('^' + _pv['dots_disallowed_in_PN'] + '$', re.VERBOSE), - "dots_allowed_in_PN": re.compile('^' + _pv['dots_allowed_in_PN'] + '$', re.VERBOSE), -} - -def _get_pv_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _pv_re["dots_allowed_in_PN"] - else: - return _pv_re["dots_disallowed_in_PN"] - def _pkgsplit(mypkg, eapi=None): """ @param mypkg: pv @@ -269,7 +300,7 @@ def _pkgsplit(mypkg, eapi=None): 1. None if input is invalid. 2. (pn, ver, rev) if input is pv """ - m = _get_pv_re(eapi).match(mypkg) + m = _get_pv_re(_get_eapi_attrs(eapi)).match(mypkg) if m is None: return None @@ -284,7 +315,7 @@ def _pkgsplit(mypkg, eapi=None): return (m.group('pn'), m.group('ver'), rev) -_cat_re = re.compile('^%s$' % _cat) +_cat_re = re.compile('^%s$' % _cat, re.UNICODE) _missing_cat = 'null' def catpkgsplit(mydata, silent=1, eapi=None): @@ -329,14 +360,25 @@ class _pkg_str(_unicode): manually convert them to a plain unicode object first. """ - def __new__(cls, cpv, slot=None, repo=None, eapi=None): + def __new__(cls, cpv, metadata=None, settings=None, eapi=None, + repo=None, slot=None): return _unicode.__new__(cls, cpv) - def __init__(self, cpv, slot=None, repo=None, eapi=None): + def __init__(self, cpv, metadata=None, settings=None, eapi=None, + repo=None, slot=None): if not isinstance(cpv, _unicode): # Avoid TypeError from _unicode.__init__ with PyPy. cpv = _unicode_decode(cpv) _unicode.__init__(cpv) + if metadata is not None: + self.__dict__['_metadata'] = metadata + slot = metadata.get('SLOT', slot) + repo = metadata.get('repository', repo) + eapi = metadata.get('EAPI', eapi) + if settings is not None: + self.__dict__['_settings'] = settings + if eapi is not None: + self.__dict__['eapi'] = eapi self.__dict__['cpv_split'] = catpkgsplit(cpv, eapi=eapi) if self.cpv_split is None: raise InvalidData(cpv) @@ -348,7 +390,25 @@ class _pkg_str(_unicode): # for match_from_list introspection self.__dict__['cpv'] = self if slot is not None: - self.__dict__['slot'] = slot + eapi_attrs = _get_eapi_attrs(eapi) + slot_match = _get_slot_re(eapi_attrs).match(slot) + if slot_match is None: + # Avoid an InvalidAtom exception when creating SLOT atoms + self.__dict__['slot'] = '0' + self.__dict__['sub_slot'] = '0' + self.__dict__['slot_invalid'] = slot + else: + if eapi_attrs.slot_operator: + slot_split = slot.split("/") + self.__dict__['slot'] = slot_split[0] + if len(slot_split) > 1: + self.__dict__['sub_slot'] = slot_split[1] + else: + self.__dict__['sub_slot'] = slot_split[0] + else: + self.__dict__['slot'] = slot + self.__dict__['sub_slot'] = slot + if repo is not None: repo = _gen_valid_repo(repo) if not repo: @@ -359,6 +419,25 @@ class _pkg_str(_unicode): raise AttributeError("_pkg_str instances are immutable", self.__class__, name, value) + @property + def stable(self): + try: + return self._stable + except AttributeError: + try: + metadata = self._metadata + settings = self._settings + except AttributeError: + raise AttributeError('stable') + if not settings.local_config: + # Since repoman uses different config instances for + # different profiles, our local instance does not + # refer to the correct profile. + raise AssertionError('invalid context') + stable = settings._isStable(self) + self.__dict__['_stable'] = stable + return stable + def pkgsplit(mypkg, silent=1, eapi=None): """ @param mypkg: either a pv or cpv diff --git a/portage_with_autodep/pym/portage/versions.pyo b/portage_with_autodep/pym/portage/versions.pyo Binary files differindex eae2743..d59719e 100644 --- a/portage_with_autodep/pym/portage/versions.pyo +++ b/portage_with_autodep/pym/portage/versions.pyo diff --git a/portage_with_autodep/pym/portage/xml/__init__.pyo b/portage_with_autodep/pym/portage/xml/__init__.pyo Binary files differindex 15f1b77..3c787a2 100644 --- a/portage_with_autodep/pym/portage/xml/__init__.pyo +++ b/portage_with_autodep/pym/portage/xml/__init__.pyo diff --git a/portage_with_autodep/pym/portage/xml/metadata.py b/portage_with_autodep/pym/portage/xml/metadata.py index 25f801a..fcd9dc0 100644 --- a/portage_with_autodep/pym/portage/xml/metadata.py +++ b/portage_with_autodep/pym/portage/xml/metadata.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Provides an easy-to-use python interface to Gentoo's metadata.xml file. @@ -28,6 +28,8 @@ 'Thomas Mills Hinkle' """ +from __future__ import unicode_literals + __all__ = ('MetaDataXML',) import sys @@ -42,18 +44,23 @@ if sys.hexversion < 0x2070000 or \ else: try: import xml.etree.cElementTree as etree - except (ImportError, SystemError): + except (SystemExit, KeyboardInterrupt): + raise + except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 import xml.etree.ElementTree as etree try: from xml.parsers.expat import ExpatError -except (ImportError, SystemError): +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): ExpatError = SyntaxError import re import xml.etree.ElementTree -import portage -from portage import os, _unicode_decode +from portage import _encodings, _unicode_encode from portage.util import unique_everseen class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): @@ -197,12 +204,13 @@ class MetaDataXML(object): self._xml_tree = None try: - self._xml_tree = etree.parse(metadata_xml_path, + self._xml_tree = etree.parse(_unicode_encode(metadata_xml_path, + encoding=_encodings['fs'], errors='strict'), parser=etree.XMLParser(target=_MetadataTreeBuilder())) except ImportError: pass except ExpatError as e: - raise SyntaxError(_unicode_decode("%s") % (e,)) + raise SyntaxError("%s" % (e,)) if isinstance(herds, etree.ElementTree): herds_etree = herds @@ -235,7 +243,8 @@ class MetaDataXML(object): if self._herdstree is None: try: - self._herdstree = etree.parse(self._herds_path, + self._herdstree = etree.parse(_unicode_encode(self._herds_path, + encoding=_encodings['fs'], errors='strict'), parser=etree.XMLParser(target=_MetadataTreeBuilder())) except (ImportError, IOError, SyntaxError): return None diff --git a/portage_with_autodep/pym/portage/xml/metadata.pyo b/portage_with_autodep/pym/portage/xml/metadata.pyo Binary files differindex 0103456..4e8910e 100644 --- a/portage_with_autodep/pym/portage/xml/metadata.pyo +++ b/portage_with_autodep/pym/portage/xml/metadata.pyo diff --git a/portage_with_autodep/pym/portage/xpak.py b/portage_with_autodep/pym/portage/xpak.py index 3262326..73f84ab 100644 --- a/portage_with_autodep/pym/portage/xpak.py +++ b/portage_with_autodep/pym/portage/xpak.py @@ -324,7 +324,7 @@ class tbz2(object): """ self.scan() # Don't care about condition... We'll rewrite the data anyway. - if break_hardlinks and self.filestat.st_nlink > 1: + if break_hardlinks and self.filestat and self.filestat.st_nlink > 1: tmp_fname = "%s.%d" % (self.file, os.getpid()) shutil.copyfile(self.file, tmp_fname) try: diff --git a/portage_with_autodep/pym/portage/xpak.pyo b/portage_with_autodep/pym/portage/xpak.pyo Binary files differindex 361a709..eec8ebc 100644 --- a/portage_with_autodep/pym/portage/xpak.pyo +++ b/portage_with_autodep/pym/portage/xpak.pyo diff --git a/portage_with_autodep/pym/repoman/__init__.pyo b/portage_with_autodep/pym/repoman/__init__.pyo Binary files differindex b443f38..e20cf84 100644 --- a/portage_with_autodep/pym/repoman/__init__.pyo +++ b/portage_with_autodep/pym/repoman/__init__.pyo diff --git a/portage_with_autodep/pym/repoman/checks.py b/portage_with_autodep/pym/repoman/checks.py index 77df603..85aa065 100644 --- a/portage_with_autodep/pym/repoman/checks.py +++ b/portage_with_autodep/pym/repoman/checks.py @@ -1,17 +1,21 @@ # repoman: Checks -# Copyright 2007-2012 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """This module contains functions used in Repoman to ascertain the quality and correctness of an ebuild.""" +from __future__ import unicode_literals + +import codecs +from itertools import chain import re import time import repoman.errors as errors import portage from portage.eapi import eapi_supports_prefix, eapi_has_implicit_rdepend, \ eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, \ - eapi_exports_AA, eapi_exports_KV + eapi_exports_AA class LineCheck(object): """Run a check on a line of an ebuild.""" @@ -66,7 +70,7 @@ class EbuildHeader(LineCheck): Copyright header errors CVS header errors License header errors - + Args: modification_year - Year the ebuild was last modified """ @@ -109,7 +113,7 @@ class EbuildWhitespace(LineCheck): ignore_line = re.compile(r'(^$)|(^(\t)*#)') ignore_comment = False leading_spaces = re.compile(r'^[\S\t]') - trailing_whitespace = re.compile(r'.*([\S]$)') + trailing_whitespace = re.compile(r'.*([\S]$)') def check(self, num, line): if self.leading_spaces.match(line) is None: @@ -159,6 +163,9 @@ class EbuildQuote(LineCheck): "GAMES_DATADIR_BASE", "GAMES_SYSCONFDIR", "GAMES_STATEDIR", "GAMES_LOGDIR", "GAMES_BINDIR"] + # variables for multibuild.eclass + var_names += ["BUILD_DIR"] + var_names = "(%s)" % "|".join(var_names) var_reference = re.compile(r'\$(\{'+var_names+'\}|' + \ var_names + '\W)') @@ -166,7 +173,7 @@ class EbuildQuote(LineCheck): r'\}?[^"\'\s]*(\s|$)') cond_begin = re.compile(r'(^|\s+)\[\[($|\\$|\s+)') cond_end = re.compile(r'(^|\s+)\]\]($|\\$|\s+)') - + def check(self, num, line): if self.var_reference.search(line) is None: return @@ -218,21 +225,13 @@ class EbuildAssignment(LineCheck): """Ensure ebuilds don't assign to readonly variables.""" repoman_check_name = 'variable.readonly' - readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=') - line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$') - ignore_line = re.compile(r'(^$)|(^(\t)*#)') - ignore_comment = False - - def __init__(self): - self.previous_line = None def check(self, num, line): match = self.readonly_assignment.match(line) e = None - if match and (not self.previous_line or not self.line_continuation.match(self.previous_line)): + if match is not None: e = errors.READONLY_ASSIGNMENT_ERROR - self.previous_line = line return e class Eapi3EbuildAssignment(EbuildAssignment): @@ -244,11 +243,11 @@ class Eapi3EbuildAssignment(EbuildAssignment): return eapi_supports_prefix(eapi) class EbuildNestedDie(LineCheck): - """Check ebuild for nested die statements (die statements in subshells""" - + """Check ebuild for nested die statements (die statements in subshells)""" + repoman_check_name = 'ebuild.nesteddie' nesteddie_re = re.compile(r'^[^#]*\s\(\s[^)]*\bdie\b') - + def check(self, num, line): if self.nesteddie_re.match(line): return errors.NESTED_DIE_ERROR @@ -293,7 +292,7 @@ class EapiDefinition(LineCheck): _eapi_re = portage._pms_eapi_re def new(self, pkg): - self._cached_eapi = pkg.metadata['EAPI'] + self._cached_eapi = pkg.eapi self._parsed_eapi = None self._eapi_line_num = None @@ -331,24 +330,6 @@ class EbuildQuotedA(LineCheck): if match: return "Quoted \"${A}\" on line: %d" -class EprefixifyDefined(LineCheck): - """ Check that prefix.eclass is inherited if needed""" - - repoman_check_name = 'eprefixify.defined' - - _eprefixify_re = re.compile(r'\beprefixify\b') - _inherit_prefix_re = re.compile(r'^\s*inherit\s(.*\s)?prefix\b') - - def new(self, pkg): - self._prefix_inherited = False - - def check(self, num, line): - if self._eprefixify_re.search(line) is not None: - if not self._prefix_inherited: - return errors.EPREFIXIFY_MISSING_INHERIT - elif self._inherit_prefix_re.search(line) is not None: - self._prefix_inherited = True - class NoOffsetWithHelpers(LineCheck): """ Check that the image location, the alternate root offset, and the offset prefix (D, ROOT, ED, EROOT and EPREFIX) are not used with @@ -401,13 +382,18 @@ class InheritDeprecated(LineCheck): # deprecated eclass : new eclass (False if no new eclass) deprecated_classes = { "bash-completion": "bash-completion-r1", + "boost-utils": False, + "distutils": "distutils-r1", "gems": "ruby-fakegem", "git": "git-2", + "mono": "mono-env", "mozconfig-2": "mozconfig-3", "mozcoreconf": "mozcoreconf-2", "php-ext-pecl-r1": "php-ext-pecl-r2", "php-ext-source-r1": "php-ext-source-r2", "php-pear": "php-pear-r1", + "python": "python-r1 / python-single-r1 / python-any-r1", + "python-distutils-ng": "python-r1 + distutils-r1", "qt3": False, "qt4": "qt4-r2", "ruby": "ruby-ng", @@ -464,63 +450,189 @@ class InheritDeprecated(LineCheck): (eclass, replacement) del self._indirect_deprecated -class InheritAutotools(LineCheck): +class InheritEclass(LineCheck): """ - Make sure appropriate functions are called in - ebuilds that inherit autotools.eclass. - """ - - repoman_check_name = 'inherit.autotools' - _inherit_autotools_re = re.compile(r'^\s*inherit\s(.*\s)?autotools(\s|$)') - _autotools_funcs = ( - "eaclocal", "eautoconf", "eautoheader", - "eautomake", "eautoreconf", "_elibtoolize") - _autotools_func_re = re.compile(r'\b(' + \ - "|".join(_autotools_funcs) + r')\b') - # Exempt eclasses: - # git - An EGIT_BOOTSTRAP variable may be used to call one of - # the autotools functions. - # subversion - An ESVN_BOOTSTRAP variable may be used to call one of - # the autotools functions. - _exempt_eclasses = frozenset(["git", "subversion"]) - - def new(self, pkg): - self._inherit_autotools = None - self._autotools_func_call = None - self._disabled = self._exempt_eclasses.intersection(pkg.inherited) - - def check(self, num, line): - if self._disabled: - return - if self._inherit_autotools is None: - self._inherit_autotools = self._inherit_autotools_re.match(line) - if self._inherit_autotools is not None and \ - self._autotools_func_call is None: - self._autotools_func_call = self._autotools_func_re.search(line) - - def end(self): - if self._inherit_autotools and self._autotools_func_call is None: - yield 'no eauto* function called' + Base class for checking for missing inherits, as well as excess inherits. -class IUseUndefined(LineCheck): - """ - Make sure the ebuild defines IUSE (style guideline - says to define IUSE even when empty). + Args: + eclass: Set to the name of your eclass. + funcs: A tuple of functions that this eclass provides. + comprehensive: Is the list of functions complete? + exempt_eclasses: If these eclasses are inherited, disable the missing + inherit check. """ - repoman_check_name = 'IUSE.undefined' - _iuse_def_re = re.compile(r'^IUSE=.*') + def __init__(self, eclass, funcs=None, comprehensive=False, + exempt_eclasses=None, ignore_missing=False, **kwargs): + self._eclass = eclass + self._comprehensive = comprehensive + self._exempt_eclasses = exempt_eclasses + self._ignore_missing = ignore_missing + inherit_re = eclass + self._inherit_re = re.compile(r'^(\s*|.*[|&]\s*)\binherit\s(.*\s)?%s(\s|$)' % inherit_re) + # Match when the function is preceded only by leading whitespace, a + # shell operator such as (, {, |, ||, or &&, or optional variable + # setting(s). This prevents false positives in things like elog + # messages, as reported in bug #413285. + self._func_re = re.compile(r'(^|[|&{(])\s*(\w+=.*)?\b(' + '|'.join(funcs) + r')\b') def new(self, pkg): - self._iuse_def = None + self.repoman_check_name = 'inherit.missing' + # We can't use pkg.inherited because that tells us all the eclasses that + # have been inherited and not just the ones we inherit directly. + self._inherit = False + self._func_call = False + if self._exempt_eclasses is not None: + inherited = pkg.inherited + self._disabled = any(x in inherited for x in self._exempt_eclasses) + else: + self._disabled = False + self._eapi = pkg.eapi def check(self, num, line): - if self._iuse_def is None: - self._iuse_def = self._iuse_def_re.match(line) + if not self._inherit: + self._inherit = self._inherit_re.match(line) + if not self._inherit: + if self._disabled or self._ignore_missing: + return + s = self._func_re.search(line) + if s is not None: + func_name = s.group(3) + eapi_func = _eclass_eapi_functions.get(func_name) + if eapi_func is None or not eapi_func(self._eapi): + self._func_call = True + return ('%s.eclass is not inherited, ' + 'but "%s" found at line: %s') % \ + (self._eclass, func_name, '%d') + elif not self._func_call: + self._func_call = self._func_re.search(line) def end(self): - if self._iuse_def is None: - yield 'IUSE is not defined' + if not self._disabled and self._comprehensive and self._inherit and not self._func_call: + self.repoman_check_name = 'inherit.unused' + yield 'no function called from %s.eclass; please drop' % self._eclass + +_eclass_eapi_functions = { + "usex" : lambda eapi: eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") +} + +# eclasses that export ${ECLASS}_src_(compile|configure|install) +_eclass_export_functions = ( + 'ant-tasks', 'apache-2', 'apache-module', 'aspell-dict', + 'autotools-utils', 'base', 'bsdmk', 'cannadic', + 'clutter', 'cmake-utils', 'db', 'distutils', 'elisp', + 'embassy', 'emboss', 'emul-linux-x86', 'enlightenment', + 'font-ebdftopcf', 'font', 'fox', 'freebsd', 'freedict', + 'games', 'games-ggz', 'games-mods', 'gdesklets', + 'gems', 'gkrellm-plugin', 'gnatbuild', 'gnat', 'gnome2', + 'gnome-python-common', 'gnustep-base', 'go-mono', 'gpe', + 'gst-plugins-bad', 'gst-plugins-base', 'gst-plugins-good', + 'gst-plugins-ugly', 'gtk-sharp-module', 'haskell-cabal', + 'horde', 'java-ant-2', 'java-pkg-2', 'java-pkg-simple', + 'java-virtuals-2', 'kde4-base', 'kde4-meta', 'kernel-2', + 'latex-package', 'linux-mod', 'mozlinguas', 'myspell', + 'myspell-r2', 'mysql', 'mysql-v2', 'mythtv-plugins', + 'oasis', 'obs-service', 'office-ext', 'perl-app', + 'perl-module', 'php-ext-base-r1', 'php-ext-pecl-r2', + 'php-ext-source-r2', 'php-lib-r1', 'php-pear-lib-r1', + 'php-pear-r1', 'python-distutils-ng', 'python', + 'qt4-build', 'qt4-r2', 'rox-0install', 'rox', 'ruby', + 'ruby-ng', 'scsh', 'selinux-policy-2', 'sgml-catalog', + 'stardict', 'sword-module', 'tetex-3', 'tetex', + 'texlive-module', 'toolchain-binutils', 'toolchain', + 'twisted', 'vdr-plugin-2', 'vdr-plugin', 'vim', + 'vim-plugin', 'vim-spell', 'virtuoso', 'vmware', + 'vmware-mod', 'waf-utils', 'webapp', 'xemacs-elisp', + 'xemacs-packages', 'xfconf', 'x-modular', 'xorg-2', + 'zproduct' +) + +_eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + + # Exempt eclasses: + # git - An EGIT_BOOTSTRAP variable may be used to call one of + # the autotools functions. + # subversion - An ESVN_BOOTSTRAP variable may be used to call one of + # the autotools functions. + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'eutils': { + 'funcs': ( + 'estack_push', 'estack_pop', 'eshopts_push', 'eshopts_pop', + 'eumask_push', 'eumask_pop', 'epatch', 'epatch_user', + 'emktemp', 'edos2unix', 'in_iuse', 'use_if_iuse', 'usex' + ), + 'comprehensive': False, + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions, + }, + + 'flag-o-matic': { + 'funcs': ( + 'filter-(ld)?flags', 'strip-flags', 'strip-unsupported-flags', + 'append-((ld|c(pp|xx)?))?flags', 'append-libs', + ), + 'comprehensive': False + }, + + 'libtool': { + 'funcs': ( + 'elibtoolize', + ), + 'comprehensive': True, + 'exempt_eclasses': ('autotools',) + }, + + 'multilib': { + 'funcs': ( + 'get_libdir', + ), + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions + ('autotools', 'libtool', + 'multilib-minimal'), + + 'comprehensive': False + }, + + 'multiprocessing': { + 'funcs': ( + 'makeopts_jobs', + ), + 'comprehensive': False + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': True + }, + + 'toolchain-funcs': { + 'funcs': ( + 'gen_usr_ldscript', + ), + 'comprehensive': False + }, + + 'user': { + 'funcs': ( + 'enewuser', 'enewgroup', + 'egetent', 'egethome', 'egetshell', 'esethome' + ), + 'comprehensive': True + } +} class EMakeParallelDisabled(PhaseCheck): """Check for emake -j1 calls which disable parallelization.""" @@ -546,8 +658,8 @@ class NoAsNeeded(LineCheck): error = errors.NO_AS_NEEDED class PreserveOldLib(LineCheck): - """Check for calls to the preserve_old_lib function.""" - repoman_check_name = 'upstream.workaround' + """Check for calls to the deprecated preserve_old_lib function.""" + repoman_check_name = 'ebuild.minorsyn' re = re.compile(r'.*preserve_old_lib') error = errors.PRESERVE_OLD_LIB @@ -665,7 +777,7 @@ class PortageInternal(LineCheck): repoman_check_name = 'portage.internal' ignore_comment = True # Match when the command is preceded only by leading whitespace or a shell - # operator such as (, {, |, ||, or &&. This prevents false postives in + # operator such as (, {, |, ||, or &&. This prevents false positives in # things like elog messages, as reported in bug #413285. re = re.compile(r'^(\s*|.*[|&{(]+\s*)\b(ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib)\b') @@ -675,25 +787,64 @@ class PortageInternal(LineCheck): if m is not None: return ("'%s'" % m.group(2)) + " called on line: %d" -_constant_checks = tuple((c() for c in ( - EbuildHeader, EbuildWhitespace, EbuildBlankLine, EbuildQuote, - EbuildAssignment, Eapi3EbuildAssignment, EbuildUselessDodoc, - EbuildUselessCdS, EbuildNestedDie, - EbuildPatches, EbuildQuotedA, EapiDefinition, EprefixifyDefined, - ImplicitRuntimeDeps, InheritAutotools, InheritDeprecated, IUseUndefined, - EMakeParallelDisabled, EMakeParallelDisabledViaMAKEOPTS, NoAsNeeded, - DeprecatedBindnowFlags, SrcUnpackPatches, WantAutoDefaultValue, - SrcCompileEconf, Eapi3DeprecatedFuncs, NoOffsetWithHelpers, - Eapi4IncompatibleFuncs, Eapi4GoneVars, BuiltWithUse, - PreserveOldLib, SandboxAddpredict, PortageInternal, - DeprecatedUseq, DeprecatedHasq))) +class PortageInternalVariableAssignment(LineCheck): + repoman_check_name = 'portage.internal' + internal_assignment = re.compile(r'\s*(export\s+)?(EXTRA_ECONF|EXTRA_EMAKE)\+?=') + + def check(self, num, line): + match = self.internal_assignment.match(line) + e = None + if match is not None: + e = 'Assignment to variable %s' % match.group(2) + e += ' on line: %d' + return e + +_base_check_classes = (InheritEclass, LineCheck, PhaseCheck) +_constant_checks = None + +def _init(experimental_inherit=False): + + global _constant_checks, _eclass_info + + if not experimental_inherit: + # Emulate the old eprefixify.defined and inherit.autotools checks. + _eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + 'ignore_missing': True, + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': False + } + } + + _constant_checks = tuple(chain((v() for k, v in globals().items() + if isinstance(v, type) and issubclass(v, LineCheck) and + v not in _base_check_classes), + (InheritEclass(k, **portage._native_kwargs(kwargs)) + for k, kwargs in _eclass_info.items()))) _here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') _ignore_comment_re = re.compile(r'^\s*#') def run_checks(contents, pkg): + unicode_escape_codec = codecs.lookup('unicode_escape') + unicode_escape = lambda x: unicode_escape_codec.decode(x)[0] + if _constant_checks is None: + _init() checks = _constant_checks here_doc_delim = None + multiline = None for lc in checks: lc.new(pkg) @@ -707,14 +858,52 @@ def run_checks(contents, pkg): here_doc = _here_doc_re.match(line) if here_doc is not None: here_doc_delim = re.compile(r'^\s*%s$' % here_doc.group(1)) + if here_doc_delim is not None: + continue + + # Unroll multiline escaped strings so that we can check things: + # inherit foo bar \ + # moo \ + # cow + # This will merge these lines like so: + # inherit foo bar moo cow + try: + # A normal line will end in the two bytes: <\> <\n>. So decoding + # that will result in python thinking the <\n> is being escaped + # and eat the single <\> which makes it hard for us to detect. + # Instead, strip the newline (which we know all lines have), and + # append a <0>. Then when python escapes it, if the line ended + # in a <\>, we'll end up with a <\0> marker to key off of. This + # shouldn't be a problem with any valid ebuild ... + line_escaped = unicode_escape(line.rstrip('\n') + '0') + except SystemExit: + raise + except: + # Who knows what kind of crazy crap an ebuild will have + # in it -- don't allow it to kill us. + line_escaped = line + if multiline: + # Chop off the \ and \n bytes from the previous line. + multiline = multiline[:-2] + line + if not line_escaped.endswith('\0'): + line = multiline + num = multinum + multiline = None + else: + continue + else: + if line_escaped.endswith('\0'): + multinum = num + multiline = line + continue - if here_doc_delim is None: - # We're not in a here-document. + if not line.endswith("#nowarn\n"): + # Finally we have a full line to parse. is_comment = _ignore_comment_re.match(line) is not None for lc in checks: if is_comment and lc.ignore_comment: continue - if lc.check_eapi(pkg.metadata['EAPI']): + if lc.check_eapi(pkg.eapi): ignore = lc.ignore_line if not ignore or not ignore.match(line): e = lc.check(num, line) diff --git a/portage_with_autodep/pym/repoman/checks.pyo b/portage_with_autodep/pym/repoman/checks.pyo Binary files differindex 1db2838..c1899fc 100644 --- a/portage_with_autodep/pym/repoman/checks.pyo +++ b/portage_with_autodep/pym/repoman/checks.pyo diff --git a/portage_with_autodep/pym/repoman/errors.py b/portage_with_autodep/pym/repoman/errors.py index 3209243..3833be6 100644 --- a/portage_with_autodep/pym/repoman/errors.py +++ b/portage_with_autodep/pym/repoman/errors.py @@ -1,7 +1,9 @@ # repoman: Error Messages -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + COPYRIGHT_ERROR = 'Invalid Gentoo Copyright on line: %d' LICENSE_ERROR = 'Invalid Gentoo/GPL License on line: %d' CVS_HEADER_ERROR = 'Malformed CVS Header on line: %d' @@ -17,9 +19,8 @@ EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS = 'Upstream parallel compilation bug (MAKEO DEPRECATED_BINDNOW_FLAGS = 'Deprecated bindnow-flags call on line: %d' EAPI_DEFINED_AFTER_INHERIT = 'EAPI defined after inherit on line: %d' NO_AS_NEEDED = 'Upstream asneeded linking bug (no-as-needed on line: %d)' -PRESERVE_OLD_LIB = 'Upstream ABI change workaround on line: %d' +PRESERVE_OLD_LIB = 'Ebuild calls deprecated preserve_old_lib on line: %d' BUILT_WITH_USE = 'built_with_use on line: %d' -EPREFIXIFY_MISSING_INHERIT = "prefix.eclass is not inherited, but eprefixify is used on line: %d" NO_OFFSET_WITH_HELPERS = "Helper function is used with D, ROOT, ED, EROOT or EPREFIX on line :%d" SANDBOX_ADDPREDICT = 'Ebuild calls addpredict on line: %d' USEQ_ERROR = 'Ebuild calls deprecated useq function on line: %d' diff --git a/portage_with_autodep/pym/repoman/errors.pyo b/portage_with_autodep/pym/repoman/errors.pyo Binary files differindex b7c2b8d..bbe757d 100644 --- a/portage_with_autodep/pym/repoman/errors.pyo +++ b/portage_with_autodep/pym/repoman/errors.pyo diff --git a/portage_with_autodep/pym/repoman/herdbase.py b/portage_with_autodep/pym/repoman/herdbase.py index 91a32cb..c5b88ff 100644 --- a/portage_with_autodep/pym/repoman/herdbase.py +++ b/portage_with_autodep/pym/repoman/herdbase.py @@ -1,18 +1,26 @@ # -*- coding: utf-8 -*- # repoman: Herd database analysis -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 or later +from __future__ import unicode_literals + import errno import xml.etree.ElementTree try: from xml.parsers.expat import ExpatError -except (ImportError, SystemError): +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 # This means that python is built without xml support. # We tolerate global scope import failures for optional # modules, so that ImportModulesTestCase can succeed (or # possibly alert us about unexpected import failures). pass + +from portage import _encodings, _unicode_encode from portage.exception import FileNotFound, ParseError, PermissionDenied __all__ = [ @@ -52,11 +60,12 @@ def make_herd_base(filename): all_emails = set() try: - xml_tree = xml.etree.ElementTree.parse(filename, + xml_tree = xml.etree.ElementTree.parse(_unicode_encode(filename, + encoding=_encodings['fs'], errors='strict'), parser=xml.etree.ElementTree.XMLParser( target=_HerdsTreeBuilder())) except ExpatError as e: - raise ParseError("metadata.xml: " + str(e)) + raise ParseError("metadata.xml: %s" % (e,)) except EnvironmentError as e: func_call = "open('%s')" % filename if e.errno == errno.EACCES: diff --git a/portage_with_autodep/pym/repoman/herdbase.pyo b/portage_with_autodep/pym/repoman/herdbase.pyo Binary files differindex 112152f..8f89411 100644 --- a/portage_with_autodep/pym/repoman/herdbase.pyo +++ b/portage_with_autodep/pym/repoman/herdbase.pyo diff --git a/portage_with_autodep/pym/repoman/utilities.py b/portage_with_autodep/pym/repoman/utilities.py index bee67aa..3ec3a4a 100644 --- a/portage_with_autodep/pym/repoman/utilities.py +++ b/portage_with_autodep/pym/repoman/utilities.py @@ -1,11 +1,11 @@ # repoman: Utilities -# Copyright 2007-2012 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """This module contains utility functions to help repoman find ebuilds to scan""" -from __future__ import print_function +from __future__ import print_function, unicode_literals __all__ = [ "detect_vcs_conflicts", @@ -25,6 +25,7 @@ __all__ = [ "UpdateChangeLog" ] +import collections import errno import io from itertools import chain @@ -33,18 +34,20 @@ import pwd import re import stat import sys +import subprocess import time import textwrap import difflib from tempfile import mkstemp +import portage from portage import os from portage import shutil -from portage import subprocess_getstatusoutput from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage import output +from portage.const import BASH_BINARY from portage.localization import _ from portage.output import red, green from portage.process import find_binary @@ -71,22 +74,31 @@ def detect_vcs_conflicts(options, vcs): Returns: None (calls sys.exit on fatal problems) """ - retval = ("","") + + cmd = None if vcs == 'cvs': logging.info("Performing a " + output.green("cvs -n up") + \ " with a little magic grep to check for updates.") - retval = subprocess_getstatusoutput("cvs -n up 2>/dev/null | " + \ + cmd = "cvs -n up 2>/dev/null | " + \ "egrep '^[^\?] .*' | " + \ - "egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'") + "egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'" if vcs == 'svn': logging.info("Performing a " + output.green("svn status -u") + \ " with a little magic grep to check for updates.") - retval = subprocess_getstatusoutput("svn status -u 2>&1 | " + \ + cmd = "svn status -u 2>&1 | " + \ "egrep -v '^. +.*/digest-[^/]+' | " + \ - "head -n-1") - - if vcs in ['cvs', 'svn']: - mylines = retval[1].splitlines() + "head -n-1" + + if cmd is not None: + # Use Popen instead of getstatusoutput(), in order to avoid + # unicode handling problems (see bug #310789). + args = [BASH_BINARY, "-c", cmd] + args = [_unicode_encode(x) for x in args] + proc = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out = _unicode_decode(proc.communicate()[0]) + proc.wait() + mylines = out.splitlines() myupdates = [] for line in mylines: if not line: @@ -98,7 +110,7 @@ def detect_vcs_conflicts(options, vcs): logging.error(red("!!! Please fix the following issues reported " + \ "from cvs: ")+green("(U,P,M,A,R,D are ok)")) logging.error(red("!!! Note: This is a pretend/no-modify pass...")) - logging.error(retval[1]) + logging.error(out) sys.exit(1) elif vcs == 'cvs' and line[0] in "UP": myupdates.append(line[2:]) @@ -298,12 +310,12 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning # we only want key value pairs where value > 0 for category, number in \ filter(lambda myitem: myitem[1] > 0, iter(stats.items())): - formatter.add_literal_data(_unicode_decode(" " + category.ljust(30))) + formatter.add_literal_data(" " + category.ljust(30)) if category in qawarnings: formatter.push_style("WARN") else: formatter.push_style("BAD") - formatter.add_literal_data(_unicode_decode(str(number))) + formatter.add_literal_data("%s" % number) formatter.pop_style() formatter.add_line_break() if not dofull: @@ -314,7 +326,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning if not full and len(fails_list) > 12: fails_list = fails_list[:12] for failure in fails_list: - formatter.add_literal_data(_unicode_decode(" " + failure)) + formatter.add_literal_data(" " + failure) formatter.add_line_break() @@ -367,10 +379,11 @@ def get_commit_message_with_editor(editor, message=None): if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK): return None try: - mylines = io.open(_unicode_encode(filename, + with io.open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace' - ).readlines() + ) as f: + mylines = f.readlines() except OSError as e: if e.errno != errno.ENOENT: raise @@ -427,7 +440,7 @@ def FindPortdir(settings): portdir = None portdir_overlay = None location = os.getcwd() - pwd = os.environ.get('PWD', '') + pwd = _unicode_decode(os.environ.get('PWD', ''), encoding=_encodings['fs']) if pwd and pwd != location and os.path.realpath(pwd) == location: # getcwd() returns the canonical path but that makes it hard for repoman to # orient itself if the user has symlinks in their portage tree structure. @@ -449,7 +462,7 @@ def FindPortdir(settings): if location[-1] != "/": location += "/" - for overlay in settings["PORTDIR_OVERLAY"].split(): + for overlay in portage.util.shlex_split(settings["PORTDIR_OVERLAY"]): overlay = os.path.realpath(overlay) try: s = os.stat(overlay) @@ -509,6 +522,28 @@ def FindPortdir(settings): return [normalize_path(x) for x in (portdir, portdir_overlay, location)] +_vcs_type = collections.namedtuple('_vcs_type', + 'name dir_name') + +_FindVCS_data = ( + _vcs_type( + name = 'git', + dir_name = '.git' + ), + _vcs_type( + name = 'bzr', + dir_name = '.bzr' + ), + _vcs_type( + name = 'hg', + dir_name = '.hg' + ), + _vcs_type( + name = 'svn', + dir_name = '.svn' + ) +) + def FindVCS(): """ Try to figure out in what VCS' working tree we are. """ @@ -520,14 +555,13 @@ def FindVCS(): pathprep = '' while depth is None or depth > 0: - if os.path.isdir(os.path.join(pathprep, '.git')): - retvcs.append('git') - if os.path.isdir(os.path.join(pathprep, '.bzr')): - retvcs.append('bzr') - if os.path.isdir(os.path.join(pathprep, '.hg')): - retvcs.append('hg') - if os.path.isdir(os.path.join(pathprep, '.svn')): # >=1.7 - retvcs.append('svn') + for vcs_type in _FindVCS_data: + vcs_dir = os.path.join(pathprep, vcs_type.dir_name) + if os.path.isdir(vcs_dir): + logging.debug('FindVCS: found %(name)s dir: %(vcs_dir)s' % + {'name': vcs_type.name, + 'vcs_dir': os.path.abspath(vcs_dir)}) + retvcs.append(vcs_type.name) if retvcs: break @@ -681,7 +715,7 @@ def get_committer_name(env=None): return user def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, - new=(), removed=(), changed=(), pretend=False): + new=(), removed=(), changed=(), pretend=False, quiet=False): """ Write an entry to an existing ChangeLog, or create a new one. Updates copyright year on changed files, and updates the header of @@ -689,8 +723,8 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, """ if '<root@' in user: - err = 'Please set ECHANGELOG_USER or run as non-root' - logging.critical(err) + if not quiet: + logging.critical('Please set ECHANGELOG_USER or run as non-root') return None # ChangeLog times are in UTC @@ -711,24 +745,13 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, old_header_lines = [] header_lines = [] + clold_file = None try: clold_file = io.open(_unicode_encode(cl_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError: - clold_file = None - - clskel_file = None - if clold_file is None: - # we will only need the ChangeLog skeleton if there is no - # ChangeLog yet - try: - clskel_file = io.open(_unicode_encode(skel_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass + pass f, clnew_path = mkstemp() @@ -736,17 +759,35 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, try: if clold_file is not None: # retain header from old ChangeLog + first_line = True for line in clold_file: - line_strip = line.strip() + line_strip = line.strip() if line_strip and line[:1] != "#": clold_lines.append(line) break + # always make sure cat/pkg is up-to-date in case we are + # moving packages around, or copied from another pkg, or ... + if first_line: + if line.startswith('# ChangeLog for'): + line = '# ChangeLog for %s/%s\n' % (category, package) + first_line = False old_header_lines.append(line) header_lines.append(_update_copyright_year(year, line)) if not line_strip: break - elif clskel_file is not None: + clskel_file = None + if not header_lines: + # delay opening this until we find we need a header + try: + clskel_file = io.open(_unicode_encode(skel_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + + if clskel_file is not None: # read skel.ChangeLog up to first empty line for line in clskel_file: line_strip = line.strip() @@ -756,7 +797,7 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, line = line.replace('<PACKAGE_NAME>', package) line = _update_copyright_year(year, line) header_lines.append(line) - header_lines.append(_unicode_decode('\n')) + header_lines.append('\n') clskel_file.close() # write new ChangeLog entry @@ -766,10 +807,10 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, if not fn.endswith('.ebuild'): continue ebuild = fn.split(os.sep)[-1][0:-7] - clnew_lines.append(_unicode_decode('*%s (%s)\n' % (ebuild, date))) + clnew_lines.append('*%s (%s)\n' % (ebuild, date)) newebuild = True if newebuild: - clnew_lines.append(_unicode_decode('\n')) + clnew_lines.append('\n') trivial_files = ('ChangeLog', 'Manifest') display_new = ['+' + elem for elem in new if elem not in trivial_files] @@ -796,11 +837,13 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, for line in textwrap.wrap(mesg, 80, \ initial_indent=' ', subsequent_indent=' ', \ break_on_hyphens=False): - clnew_lines.append(_unicode_decode('%s\n' % line)) + clnew_lines.append('%s\n' % line) for line in textwrap.wrap(msg, 80, \ initial_indent=' ', subsequent_indent=' '): - clnew_lines.append(_unicode_decode('%s\n' % line)) - clnew_lines.append(_unicode_decode('\n')) + clnew_lines.append('%s\n' % line) + # Don't append a trailing newline if the file is new. + if clold_file is not None: + clnew_lines.append('\n') f = io.open(f, mode='w', encoding=_encodings['repo.content'], errors='backslashreplace') @@ -832,17 +875,20 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, # in the unified_diff call below. clold_lines = old_header_lines + clold_lines - for line in clold_file: - f.write(line) + # Trim any trailing newlines. + lines = clold_file.readlines() clold_file.close() + while lines and lines[-1] == '\n': + del lines[-1] + f.writelines(lines) f.close() - # show diff (do we want to keep on doing this, or only when - # pretend?) - for line in difflib.unified_diff(clold_lines, clnew_lines, - fromfile=cl_path, tofile=cl_path, n=0): - util.writemsg_stdout(line, noiselevel=-1) - util.writemsg_stdout("\n", noiselevel=-1) + # show diff + if not quiet: + for line in difflib.unified_diff(clold_lines, clnew_lines, + fromfile=cl_path, tofile=cl_path, n=0): + util.writemsg_stdout(line, noiselevel=-1) + util.writemsg_stdout("\n", noiselevel=-1) if pretend: # remove what we've done diff --git a/portage_with_autodep/pym/repoman/utilities.pyo b/portage_with_autodep/pym/repoman/utilities.pyo Binary files differindex c9f6734..1956722 100644 --- a/portage_with_autodep/pym/repoman/utilities.pyo +++ b/portage_with_autodep/pym/repoman/utilities.pyo diff --git a/src/hook_lib/file_hook.c b/src/hook_lib/file_hook.c index 493c206..390efce 100644 --- a/src/hook_lib/file_hook.c +++ b/src/hook_lib/file_hook.c @@ -540,7 +540,7 @@ pid_t fork(void) { __fixenv(); // We have to get lock here because we can't be sure that other thread not in - // protected code. When we forks, only current thread gets copied, so + // protected code. When someone calls fork, only current thread gets copied, so // child mutex may be always locked without this. pthread_mutex_lock( &socketblock ); |