diff options
Diffstat (limited to 'pym/gentoolkit/eclean')
| -rw-r--r-- | pym/gentoolkit/eclean/__init__.py | 4 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/clean.py | 149 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/cli.py | 500 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/exclude.py | 262 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/output.py | 179 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/pkgindex.py | 89 | ||||
| -rw-r--r-- | pym/gentoolkit/eclean/search.py | 520 |
7 files changed, 1703 insertions, 0 deletions
diff --git a/pym/gentoolkit/eclean/__init__.py b/pym/gentoolkit/eclean/__init__.py new file mode 100644 index 0000000..edf9385 --- /dev/null +++ b/pym/gentoolkit/eclean/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python +# +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py new file mode 100644 index 0000000..9f6d597 --- /dev/null +++ b/pym/gentoolkit/eclean/clean.py @@ -0,0 +1,149 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +import sys + +from portage import os +import gentoolkit.pprinter as pp +from gentoolkit.eclean.pkgindex import PkgIndex + + +class CleanUp(object): + """Performs all cleaning actions to distfiles or package directories. + + @param controller: a progress output/user interaction controller function + which returns a Boolean to control file deletion + or bypassing/ignoring + """ + + def __init__(self, controller): + self.controller = controller + + def clean_dist(self, clean_dict): + """Calculate size of each entry for display, prompt user if needed, + delete files if approved and return the total size of files that + have been deleted. + + @param clean_dict: dictionary of {'display name':[list of files]} + + @rtype: int + @return: total size that was cleaned + """ + file_type = 'file' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # clean all entries one by one + for key in clean_keys: + clean_size += self._clean_files(clean_dict[key], key, file_type) + # return total size of deleted or to delete files + return clean_size + + def clean_pkgs(self, clean_dict, pkgdir): + """Calculate size of each entry for display, prompt user if needed, + delete files if approved and return the total size of files that + have been deleted. + + @param clean_dict: dictionary of {'display name':[list of files]} + @param metadata: package index of type portage.getbinpkg.PackageIndex() + @param pkgdir: path to the package directory to be cleaned + + @rtype: int + @return: total size that was cleaned + """ + file_type = 'binary package' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # clean all entries one by one + for key in clean_keys: + clean_size += self._clean_files(clean_dict[key], key, file_type) + + # run 'emaint --fix' here + if clean_size: + index_control = PkgIndex(self.controller) + # emaint is not yet importable so call it + # print a blank line here for separation + print() + clean_size += index_control.call_emaint() + # return total size of deleted or to delete files + return clean_size + + + def pretend_clean(self, clean_dict): + """Shortcut function that calculates total space savings + for the files in clean_dict. + + @param clean_dict: dictionary of {'display name':[list of files]} + @rtype: integer + @return: total size that would be cleaned + """ + file_type = 'file' + clean_keys = self._sort_keys(clean_dict) + clean_size = 0 + # tally all entries one by one + for key in clean_keys: + key_size = self._get_size(clean_dict[key]) + self.controller(key_size, key, clean_dict[key], file_type) + clean_size += key_size + return clean_size + + def _get_size(self, key): + """Determine the total size for an entry (may be several files).""" + key_size = 0 + for file_ in key: + #print file_ + # get total size for an entry (may be several files, and + # links don't count + # ...get its statinfo + try: + statinfo = os.stat(file_) + if statinfo.st_nlink == 1: + key_size += statinfo.st_size + except EnvironmentError as er: + print( pp.error( + "Could not get stat info for:" + file_), file=sys.stderr) + print( pp.error("Error: %s" %str(er)), file=sys.stderr) + return key_size + + def _sort_keys(self, clean_dict): + """Returns a list of sorted dictionary keys.""" + # sorting helps reading + clean_keys = sorted(clean_dict) + return clean_keys + + def _clean_files(self, files, key, file_type): + """File removal function.""" + clean_size = 0 + for file_ in files: + #print file_, type(file_) + # ...get its statinfo + try: + statinfo = os.stat(file_) + except EnvironmentError as er: + print( pp.error( + "Could not get stat info for:" + file_), file=sys.stderr) + print( pp.error( + "Error: %s" %str(er)), file=sys.stderr) + if self.controller(statinfo.st_size, key, file_, file_type): + # ... try to delete it. + try: + os.unlink(file_) + # only count size if successfully deleted and not a link + if statinfo.st_nlink == 1: + clean_size += statinfo.st_size + except EnvironmentError as er: + print( pp.error("Could not delete "+file_), file=sys.stderr) + print( pp.error("Error: %s" %str(er)), file=sys.stderr) + return clean_size + + + + + + + diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py new file mode 100644 index 0000000..6a507ef --- /dev/null +++ b/pym/gentoolkit/eclean/cli.py @@ -0,0 +1,500 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +__author__ = "Thomas de Grenier de Latour (tgl), " + \ + "modular re-write by: Brian Dolbec (dol-sen)" +__email__ = "degrenier@easyconnect.fr, " + \ + "brian.dolbec@gmail.com" +__version__ = "svn" +__productname__ = "eclean" +__description__ = "A cleaning tool for Gentoo distfiles and binaries." + + +import sys +import re +import time +import getopt + +import portage +from portage import os +from portage.output import white, yellow, turquoise, green, teal, red + +import gentoolkit.pprinter as pp +from gentoolkit.eclean.search import (DistfilesSearch, + findPackages, port_settings, pkgdir) +from gentoolkit.eclean.exclude import (parseExcludeFile, + ParseExcludeFileException) +from gentoolkit.eclean.clean import CleanUp +from gentoolkit.eclean.output import OutputControl +#from gentoolkit.eclean.dbapi import Dbapi + +def printVersion(): + """Output the version info.""" + print( "%s (%s) - %s" \ + % (__productname__, __version__, __description__)) + print() + print("Author: %s <%s>" % (__author__,__email__)) + print("Copyright 2003-2009 Gentoo Foundation") + print("Distributed under the terms of the GNU General Public License v2") + + +def printUsage(_error=None, help=None): + """Print help message. May also print partial help to stderr if an + error from {'options','actions'} is specified.""" + + out = sys.stdout + if _error: + out = sys.stderr + if not _error in ('actions', 'global-options', \ + 'packages-options', 'distfiles-options', \ + 'merged-packages-options', 'merged-distfiles-options', \ + 'time', 'size'): + _error = None + if not _error and not help: help = 'all' + if _error == 'time': + print( pp.error("Wrong time specification"), file=out) + print( "Time specification should be an integer followed by a"+ + " single letter unit.", file=out) + print( "Available units are: y (years), m (months), w (weeks), "+ + "d (days) and h (hours).", file=out) + print( "For instance: \"1y\" is \"one year\", \"2w\" is \"two"+ + " weeks\", etc. ", file=out) + return + if _error == 'size': + print( pp.error("Wrong size specification"), file=out) + print( "Size specification should be an integer followed by a"+ + " single letter unit.", file=out) + print( "Available units are: G, M, K and B.", file=out) + print("For instance: \"10M\" is \"ten megabytes\", \"200K\" "+ + "is \"two hundreds kilobytes\", etc.", file=out) + return + if _error in ('global-options', 'packages-options', 'distfiles-options', \ + 'merged-packages-options', 'merged-distfiles-options',): + print( pp.error("Wrong option on command line."), file=out) + print( file=out) + elif _error == 'actions': + print( pp.error("Wrong or missing action name on command line."), file=out) + print( file=out) + print( white("Usage:"), file=out) + if _error in ('actions','global-options', 'packages-options', \ + 'distfiles-options') or help == 'all': + print( " "+turquoise(__productname__), + yellow("[global-option] ..."), + green("<action>"), + yellow("[action-option] ..."), file=out) + if _error == 'merged-distfiles-options' or help in ('all','distfiles'): + print( " "+turquoise(__productname__+'-dist'), + yellow("[global-option, distfiles-option] ..."), file=out) + if _error == 'merged-packages-options' or help in ('all','packages'): + print( " "+turquoise(__productname__+'-pkg'), + yellow("[global-option, packages-option] ..."), file=out) + if _error in ('global-options', 'actions'): + print( " "+turquoise(__productname__), + yellow("[--help, --version]"), file=out) + if help == 'all': + print( " "+turquoise(__productname__+"(-dist,-pkg)"), + yellow("[--help, --version]"), file=out) + if _error == 'merged-packages-options' or help == 'packages': + print( " "+turquoise(__productname__+'-pkg'), + yellow("[--help, --version]"), file=out) + if _error == 'merged-distfiles-options' or help == 'distfiles': + print( " "+turquoise(__productname__+'-dist'), + yellow("[--help, --version]"), file=out) + print(file=out) + if _error in ('global-options', 'merged-packages-options', \ + 'merged-distfiles-options') or help: + print( "Available global", yellow("options")+":", file=out) + print( yellow(" -C, --nocolor")+ + " - turn off colors on output", file=out) + print( yellow(" -d, --destructive")+ + " - only keep the minimum for a reinstallation", file=out) + print( yellow(" -e, --exclude-file=<path>")+ + " - path to the exclusion file", file=out) + print( yellow(" -i, --interactive")+ + " - ask confirmation before deletions", file=out) + print( yellow(" -n, --package-names")+ + " - protect all versions (when --destructive)", file=out) + print( yellow(" -p, --pretend")+ + " - only display what would be cleaned", file=out) + print( yellow(" -q, --quiet")+ + " - be as quiet as possible", file=out) + print( yellow(" -t, --time-limit=<time>")+ + " - don't delete files modified since "+yellow("<time>"), file=out) + print( " "+yellow("<time>"), "is a duration: \"1y\" is"+ + " \"one year\", \"2w\" is \"two weeks\", etc. ", file=out) + print( " "+"Units are: y (years), m (months), w (weeks), "+ + "d (days) and h (hours).", file=out) + print( yellow(" -h, --help")+ \ + " - display the help screen", file=out) + print( yellow(" -V, --version")+ + " - display version info", file=out) + print( file=out) + if _error == 'actions' or help == 'all': + print( "Available", green("actions")+":", file=out) + print( green(" packages")+ + " - clean outdated binary packages from PKGDIR", file=out) + print( green(" distfiles")+ + " - clean outdated packages sources files from DISTDIR", file=out) + print( file=out) + if _error in ('packages-options','merged-packages-options') \ + or help in ('all','packages'): + print( "Available", yellow("options"),"for the", + green("packages"),"action:", file=out) + print( yellow(" NONE :)"), file=out) + print( file=out) + if _error in ('distfiles-options', 'merged-distfiles-options') \ + or help in ('all','distfiles'): + print("Available", yellow("options"),"for the", + green("distfiles"),"action:", file=out) + print( yellow(" -f, --fetch-restricted")+ + " - protect fetch-restricted files (when --destructive)", file=out) + print( yellow(" -s, --size-limit=<size>")+ + " - don't delete distfiles bigger than "+yellow("<size>"), file=out) + print( " "+yellow("<size>"), "is a size specification: "+ + "\"10M\" is \"ten megabytes\", \"200K\" is", file=out) + print( " "+"\"two hundreds kilobytes\", etc. Units are: "+ + "G, M, K and B.", file=out) + print( file=out) + print( "More detailed instruction can be found in", + turquoise("`man %s`" % __productname__), file=out) + + +class ParseArgsException(Exception): + """For parseArgs() -> main() communications.""" + def __init__(self, value): + self.value = value # sdfgsdfsdfsd + def __str__(self): + return repr(self.value) + + +def parseSize(size): + """Convert a file size "Xu" ("X" is an integer, and "u" in + [G,M,K,B]) into an integer (file size in Bytes). + + @raise ParseArgsException: in case of failure + """ + units = { + 'G': (1024**3), + 'M': (1024**2), + 'K': 1024, + 'B': 1 + } + try: + match = re.match(r"^(?P<value>\d+)(?P<unit>[GMKBgmkb])?$",size) + size = int(match.group('value')) + if match.group('unit'): + size *= units[match.group('unit').capitalize()] + except: + raise ParseArgsException('size') + return size + + +def parseTime(timespec): + """Convert a duration "Xu" ("X" is an int, and "u" a time unit in + [Y,M,W,D,H]) into an integer which is a past EPOCH date. + Raises ParseArgsException('time') in case of failure. + (yep, big approximations inside... who cares?). + """ + units = {'H' : (60 * 60)} + units['D'] = units['H'] * 24 + units['W'] = units['D'] * 7 + units['M'] = units['D'] * 30 + units['Y'] = units['D'] * 365 + try: + # parse the time specification + match = re.match(r"^(?P<value>\d+)(?P<unit>[YMWDHymwdh])?$",timespec) + value = int(match.group('value')) + if not match.group('unit'): unit = 'D' + else: unit = match.group('unit').capitalize() + except: + raise ParseArgsException('time') + return time.time() - (value * units[unit]) + + +def parseArgs(options={}): + """Parse the command line arguments. Raise exceptions on + errors or non-action modes (help/version). Returns an action, and affect + the options dict. + """ + + def optionSwitch(option,opts,action=None): + """local function for interpreting command line options + and setting options accordingly""" + return_code = True + for o, a in opts: + if o in ("-h", "--help"): + if action: + raise ParseArgsException('help-'+action) + else: + raise ParseArgsException('help') + elif o in ("-V", "--version"): + raise ParseArgsException('version') + elif o in ("-C", "--nocolor"): + options['nocolor'] = True + pp.output.nocolor() + elif o in ("-d", "--destructive"): + options['destructive'] = True + elif o in ("-D", "--deprecated"): + options['deprecated'] = True + elif o in ("-i", "--interactive") and not options['pretend']: + options['interactive'] = True + elif o in ("-p", "--pretend"): + options['pretend'] = True + options['interactive'] = False + elif o in ("-q", "--quiet"): + options['quiet'] = True + options['verbose'] = False + elif o in ("-t", "--time-limit"): + options['time-limit'] = parseTime(a) + elif o in ("-e", "--exclude-file"): + options['exclude-file'] = a + elif o in ("-n", "--package-names"): + options['package-names'] = True + elif o in ("-f", "--fetch-restricted"): + options['fetch-restricted'] = True + elif o in ("-s", "--size-limit"): + options['size-limit'] = parseSize(a) + elif o in ("-v", "--verbose") and not options['quiet']: + options['verbose'] = True + else: + return_code = False + # sanity check of --destructive only options: + for opt in ('fetch-restricted', 'package-names'): + if (not options['destructive']) and options[opt]: + if not options['quiet']: + print( pp.error( + "--%s only makes sense in --destructive mode." % opt), file=sys.stderr) + options[opt] = False + return return_code + + # here are the different allowed command line options (getopt args) + getopt_options = {'short':{}, 'long':{}} + getopt_options['short']['global'] = "CdDipqe:t:nhVv" + getopt_options['long']['global'] = ["nocolor", "destructive", + "deprecated", "interactive", "pretend", "quiet", "exclude-file=", + "time-limit=", "package-names", "help", "version", "verbose"] + getopt_options['short']['distfiles'] = "fs:" + getopt_options['long']['distfiles'] = ["fetch-restricted", "size-limit="] + getopt_options['short']['packages'] = "" + getopt_options['long']['packages'] = [""] + # set default options, except 'nocolor', which is set in main() + options['interactive'] = False + options['pretend'] = False + options['quiet'] = False + options['accept_all'] = False + options['destructive'] = False + options['deprecated'] = False + options['time-limit'] = 0 + options['package-names'] = False + options['fetch-restricted'] = False + options['size-limit'] = 0 + options['verbose'] = False + # if called by a well-named symlink, set the acction accordingly: + action = None + # temp print line to ensure it is the svn/branch code running, etc.. + #print( "###### svn/branch/gentoolkit_eclean ####### ==> ", os.path.basename(sys.argv[0])) + if os.path.basename(sys.argv[0]) in \ + (__productname__+'-pkg', __productname__+'-packages'): + action = 'packages' + elif os.path.basename(sys.argv[0]) in \ + (__productname__+'-dist', __productname__+'-distfiles'): + action = 'distfiles' + # prepare for the first getopt + if action: + short_opts = getopt_options['short']['global'] \ + + getopt_options['short'][action] + long_opts = getopt_options['long']['global'] \ + + getopt_options['long'][action] + opts_mode = 'merged-'+action + else: + short_opts = getopt_options['short']['global'] + long_opts = getopt_options['long']['global'] + opts_mode = 'global' + # apply getopts to command line, show partial help on failure + try: + opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) + except: + raise ParseArgsException(opts_mode+'-options') + # set options accordingly + optionSwitch(options,opts,action=action) + # if action was already set, there should be no more args + if action and len(args): + raise ParseArgsException(opts_mode+'-options') + # if action was set, there is nothing left to do + if action: + return action + # So, we are in "eclean --foo action --bar" mode. Parse remaining args... + # Only two actions are allowed: 'packages' and 'distfiles'. + if not len(args) or not args[0] in ('packages','distfiles'): + raise ParseArgsException('actions') + action = args.pop(0) + # parse the action specific options + try: + opts, args = getopt.getopt(args, \ + getopt_options['short'][action], \ + getopt_options['long'][action]) + except: + raise ParseArgsException(action+'-options') + # set options again, for action-specific options + optionSwitch(options,opts,action=action) + # any remaning args? Then die! + if len(args): + raise ParseArgsException(action+'-options') + # returns the action. Options dictionary is modified by side-effect. + return action + + +def doAction(action,options,exclude={}, output=None): + """doAction: execute one action, ie display a few message, call the right + find* function, and then call doCleanup with its result.""" + # define vocabulary for the output + if action == 'packages': + files_type = "binary packages" + else: + files_type = "distfiles" + saved = {} + deprecated = {} + # find files to delete, depending on the action + if not options['quiet']: + output.einfo("Building file list for "+action+" cleaning...") + if action == 'packages': + clean_me = findPackages( + options, + exclude=exclude, + destructive=options['destructive'], + package_names=options['package-names'], + time_limit=options['time-limit'], + pkgdir=pkgdir, + #port_dbapi=Dbapi(portage.db[portage.root]["porttree"].dbapi), + #var_dbapi=Dbapi(portage.db[portage.root]["vartree"].dbapi), + ) + else: + # accept defaults + engine = DistfilesSearch(output=options['verbose-output'], + #portdb=Dbapi(portage.db[portage.root]["porttree"].dbapi), + #var_dbapi=Dbapi(portage.db[portage.root]["vartree"].dbapi), + ) + clean_me, saved, deprecated = engine.findDistfiles( + exclude=exclude, + destructive=options['destructive'], + fetch_restricted=options['fetch-restricted'], + package_names=options['package-names'], + time_limit=options['time-limit'], + size_limit=options['size-limit'], + deprecate = options['deprecated'] + ) + # actually clean files if something was found + if clean_me: + # verbose pretend message + if options['pretend'] and not options['quiet']: + output.einfo("Here are the "+files_type+" that would be deleted:") + # verbose non-pretend message + elif not options['quiet']: + output.einfo("Cleaning " + files_type +"...") + # do the cleanup, and get size of deleted files + cleaner = CleanUp( output.progress_controller) + if options['pretend']: + clean_size = cleaner.pretend_clean(clean_me) + elif action in ['distfiles']: + clean_size = cleaner.clean_dist(clean_me) + elif action in ['packages']: + clean_size = cleaner.clean_pkgs(clean_me, + pkgdir) + # vocabulary for final message + if options['pretend']: + verb = "would be" + else: + verb = "were" + # display freed space + if not options['quiet']: + output.total('normal', clean_size, len(clean_me), verb, action) + # nothing was found, return + elif not options['quiet']: + output.einfo("Your "+action+" directory was already clean.") + if saved and not options['quiet']: + print() + print( (pp.emph(" The folowing ") + yellow("Deprecated") + + pp.emph(" files were saved from cleaning due to exclusion file entries"))) + output.set_colors('deprecated') + clean_size = cleaner.pretend_clean(saved) + output.total('deprecated', clean_size, len(saved), verb, action) + if deprecated and not options['quiet']: + print() + print( (pp.emph(" The folowing ") + yellow("Deprecated") + + pp.emph(" installed packages were found"))) + output.set_colors('deprecated') + output.list_pkgs(deprecated) + + +def main(): + """Parse command line and execute all actions.""" + # set default options + options = {} + options['nocolor'] = (port_settings["NOCOLOR"] in ('yes','true') + or not sys.stdout.isatty()) + if options['nocolor']: + pp.output.nocolor() + # parse command line options and actions + try: + action = parseArgs(options) + # filter exception to know what message to display + except ParseArgsException as e: + if e.value == 'help': + printUsage(help='all') + sys.exit(0) + elif e.value[:5] == 'help-': + printUsage(help=e.value[5:]) + sys.exit(0) + elif e.value == 'version': + printVersion() + sys.exit(0) + else: + printUsage(e.value) + sys.exit(2) + output = OutputControl(options) + options['verbose-output'] = lambda x: None + if not options['quiet']: + if options['verbose']: + options['verbose-output'] = output.einfo + # parse the exclusion file + if 'exclude-file' in options: + try: + exclude = parseExcludeFile(options['exclude-file'], + options['verbose-output']) + except ParseExcludeFileException as e: + print( pp.error(str(e)), file=sys.stderr) + print( pp.error( + "Invalid exclusion file: %s" % options['exclude-file']), file=sys.stderr) + print( pp.error( + "See format of this file in `man %s`" % __productname__), file=sys.stderr) + sys.exit(1) + else: + exclude_file = "/etc/%s/%s.exclude" % (__productname__ , action) + if os.path.isfile(exclude_file): + options['exclude-file'] = exclude_file + exclude={} + # security check for non-pretend mode + if not options['pretend'] and portage.secpass == 0: + print( pp.error( + "Permission denied: you must be root or belong to " + + "the portage group."), file=sys.stderr) + sys.exit(1) + # execute action + doAction(action, options, exclude=exclude, + output=output) + + +if __name__ == "__main__": + """actually call main() if launched as a script""" + try: + main() + except KeyboardInterrupt: + print( "Aborted.") + sys.exit(130) + sys.exit(0) diff --git a/pym/gentoolkit/eclean/exclude.py b/pym/gentoolkit/eclean/exclude.py new file mode 100644 index 0000000..74a982a --- /dev/null +++ b/pym/gentoolkit/eclean/exclude.py @@ -0,0 +1,262 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +import sys +import re +import portage + +from portage import os +from gentoolkit.pprinter import warn + +# Misc. shortcuts to some portage stuff: +listdir = portage.listdir + +FILENAME_RE = [re.compile(r'(?P<pkgname>[-a-zA-z0-9\+]+)(?P<ver>-\d+\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z]+)(?P<ver>_\d+\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z_]+)(?P<ver>\d\d+\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z0-9_]+)(?P<ver>-default\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z0-9]+)(?P<ver>_\d\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z0-9\+\.]+)(?P<ver>-\d+\S+)'), + re.compile(r'(?P<pkgname>[-a-zA-z0-9\+\.]+)(?P<ver>.\d+\S+)')] + +debug_modules = [] + +def dprint(module, message): + if module in debug_modules: + print(message) + +def isValidCP(cp): + """Check whether a string is a valid cat/pkg-name. + + This is for 2.0.51 vs. CVS HEAD compatibility, I've not found any function + for that which would exists in both. Weird... + + @param cp: catageory/package string + @rtype: bool + """ + + if not '/' in cp: + return False + try: + portage.cpv_getkey(cp+"-0") + except: + return False + else: + return True + + +class ParseExcludeFileException(Exception): + """For parseExcludeFile() -> main() communication. + + @param value: Error message string + """ + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + + +def parseExcludeFile(filepath, output): + """Parses an exclusion file. + + @param filepath: file containing the list of cat/pkg's to exclude + @param output: --verbose enabled output method or "lambda x: None" + + @rtype: dict + @return: an exclusion dict + @raise ParseExcludeFileException: in case of fatal error + """ + + exclude = { + 'categories': {}, + 'packages': {}, + 'anti-packages': {}, + 'filenames': {} + } + output("Parsing Exclude file: " + filepath) + try: + file_ = open(filepath,"r") + except IOError: + raise ParseExcludeFileException("Could not open exclusion file: " + + filepath) + filecontents = file_.readlines() + file_.close() + cat_re = re.compile('^(?P<cat>[a-zA-Z0-9]+-[a-zA-Z0-9]+)(/\*)?$') + cp_re = re.compile('^(?P<cp>[-a-zA-Z0-9_]+/[-a-zA-Z0-9_]+)$') + # used to output the line number for exception error reporting + linenum = 0 + for line in filecontents: + # need to increment it here due to continue statements. + linenum += 1 + line = line.strip() + if not len(line): # skip blank a line + continue + if line[0] == '#': # skip a comment line + continue + #print( "parseExcludeFile: line=", line) + try: # category matching + cat = cat_re.match(line).group('cat') + #print( "parseExcludeFile: found cat=", cat) + except: + pass + else: + if not cat in portage.settings.categories: + raise ParseExcludeFileException("Invalid category: "+cat + + " @line # " + str(linenum)) + exclude['categories'][cat] = None + continue + dict_key = 'packages' + if line[0] == '!': # reverses category setting + dict_key = 'anti-packages' + line = line[1:] + try: # cat/pkg matching + cp = cp_re.match(line).group('cp') + #print( "parseExcludeFile: found cp=", cp) + if isValidCP(cp): + exclude[dict_key][cp] = None + continue + else: + raise ParseExcludeFileException("Invalid cat/pkg: "+cp + + " @line # " + str(linenum)) + except: + pass + #raise ParseExcludeFileException("Invalid line: "+line) + try: # filename matching. + exclude['filenames'][line] = re.compile(line) + #print( "parseExcludeFile: found filenames", line) + except: + try: + exclude['filenames'][line] = re.compile(re.escape(line)) + #print( "parseExcludeFile: found escaped filenames", line) + except: + raise ParseExcludeFileException("Invalid file name/regular " + + "expression: @line # " + str(linenum) + " line=" +line) + output("Exclude file parsed. Found " + + "%d categories, %d packages, %d anti-packages %d filenames" + %(len(exclude['categories']), len(exclude['packages']), + len(exclude['anti-packages']), len(exclude['filenames']))) + #print() + #print( "parseExcludeFile: final exclude_dict = ", exclude) + #print() + return exclude + +def cp_all(categories): + """temp function until the new portdb.cp_all([cat,...]) + behaviour is fully available. + + @param categories: list of categories to get all packages for + eg. ['app-portage', 'sys-apps',...] + @rtype: list of cat/pkg's ['foo/bar', 'foo/baz'] + """ + try: + cps = portage.portdb.cp_all(categories) + message = "Deprication Warning: eclean.exclude.cp_all()\n" + \ + "New portage functionality is available " +\ + "Please migrate code permanently" + print( warn(message), file=sys.stderr) + except: # new behaviour not available + #~ message = "Exception: eclean.exclude.cp_all() " +\ + #~ "new portdb.cp_all() behavior not found. using fallback code" + #~ print( warn(message), file=sys.stderr) + cps = [] + # XXX: i smell an access to something which is really out of API... + _pkg_dir_name_re = re.compile(r'^\w[-+\w]*$') + for tree in portage.portdb.porttrees: + for cat in categories: + for pkg in listdir(os.path.join(tree,cat), + EmptyOnError=1, ignorecvs=1, dirsonly=1): + if not _pkg_dir_name_re.match(pkg) or pkg == "CVS": + continue + cps.append(cat+'/'+pkg) + #print( "cp_all: new cps list=", cps) + return cps + +def exclDictExpand(exclude): + """Returns a dictionary of all CP/CPV from porttree which match + the exclusion dictionary. + """ + d = {} + if 'categories' in exclude: + # replace the following cp_all call with + # portage.portdb.cp_all([cat1, cat2]) + # when it is available in all portage versions. + cps = cp_all(exclude['categories']) + for cp in cps: + d[cp] = None + if 'packages' in exclude: + for cp in exclude['packages']: + d[cp] = None + if 'anti-packages' in exclude: + for cp in exclude['anti-packages']: + if cp in d: + del d[cp] + return d + +def exclDictMatchCP(exclude,pkg): + """Checks whether a CP matches the exclusion rules.""" + if 'anti-packages' in exclude and pkg in exclude['anti-packages']: + return False + if 'packages' in exclude and pkg in exclude['packages']: + return True + cat = pkg.split('/')[0] + if 'categories' in exclude and cat in exclude['categories']: + return True + return False + +def exclDictExpandPkgname(exclude): + """Returns a set of all pkgnames from porttree which match + the exclusion dictionary. + """ + p = set() + if 'categories' in exclude: + # replace the following cp_all call with + # portage.portdb.cp_all([cat1, cat2]) + # when it is available in all portage versions. + cps = cp_all(exclude['categories']) + for cp in cps: + pkgname = cp.split('/')[1] + p.add(pkgname) + if 'packages' in exclude: + pkgname = cp.split('/')[1] + p.add(pkgname) + if 'anti-packages' in exclude: + for cp in exclude['anti-packages']: + if cp in p: + p.remove(cp) + return p + + +def exclMatchFilename(exclude_names, filename): + """Attempts to split the package name out of a filename + and then checks if it matches any exclusion rules. + + This is intended to be run on the cleaning list after all + normal checks and removal of protected files. This will reduce + the number of files to perform this last minute check on + + @param exclude_names: a set of pkgnames to exlcude + @param filename: + + @rtype: bool + """ + found = False + index = 0 + while not found and index < len(FILENAME_RE): + found = FILENAME_RE[index].match(filename) + index += 1 + if not found: + dprint( "exclude", "exclMatchFilename: filename: " +\ + "%s, Could not determine package name" %filename) + return False + pkgname = found.group('pkgname') + dprint("exclude", "exclMatchFilename: found pkgname = " + + "%s, %s, %d, %s" %(pkgname, str(pkgname in exclude_names), + index-1, filename)) + return (pkgname in exclude_names) + diff --git a/pym/gentoolkit/eclean/output.py b/pym/gentoolkit/eclean/output.py new file mode 100644 index 0000000..670e67c --- /dev/null +++ b/pym/gentoolkit/eclean/output.py @@ -0,0 +1,179 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +import sys +import portage +from portage.output import * +from gentoolkit.pprinter import cpv, number, emph + + +class OutputControl(object): + """Outputs data according to predetermined options and handles any user + interaction. + + @param options: dictionary of boolean options as determined in cli.py + used here: interactive, pretend, quiet, accept_all, nocolor. + """ + + def __init__(self, options): + if not options: + # set some defaults + self.options['interactive'] = False + self.options['pretend'] = True + self.options['quiet'] = False + self.options['accept_all'] = False + self.options['nocolor'] = False + else: + self.options = options + self.set_colors("normal") + + def set_colors(self, mode): + """Sets the colors for the progress_controller + and prettysize output + + @param mode: string, 1 of ["normal", "deprecated"] + """ + if mode == "normal": + self.pkg_color = cpv # green + self.numbers = number # turquoise + self.brace = blue + elif mode == "deprecated": + self.pkg_color = yellow + self.numbers = teal # darkgreen + self.brace = blue + + def einfo(self, message=""): + """Display an info message depending on a color mode. + + @param message: text string to display + + @outputs to stdout. + """ + if not self.options['nocolor']: + prefix = " "+green('*') + else: + prefix = ">>>" + print(prefix,message) + + def eprompt(self, message): + """Display a user question depending on a color mode. + + @param message: text string to display + + @output to stdout + """ + if not self.options['nocolor']: + prefix = " "+red('>')+" " + else: + prefix = "??? " + sys.stdout.write(prefix+message) + sys.stdout.flush() + + def prettySize(self, size, justify=False, color=None): + """int -> byte/kilo/mega/giga converter. Optionally + justify the result. Output is a string. + + @param size: integer + @param justify: optional boolean, defaults to False + @param color: optional color, defaults to green + as defined in portage.output + + @returns a formatted and (escape sequenced) + colorized text string + """ + if color == None: + color = self.numbers + units = [" G"," M"," K"," B"] + approx = 0 + while len(units) and size >= 1000: + approx = 1 + size = size / 1024. + units.pop() + sizestr = '%d'% size + units[-1] + if justify: + sizestr = " " + self.brace("[ ") + \ + color(sizestr.rjust(7)) + self.brace(" ]") + return sizestr + + def yesNoAllPrompt(self, message="Do you want to proceed?"): + """Print a prompt until user answer in yes/no/all. Return a + boolean for answer, and also may affect the 'accept_all' option. + + @param message: optional different input string from the default + message of: "Do you want to proceed?" + @outputs to stdout + @modifies class var options['accept_all'] + @rtype: bool + """ + user_string="xxx" + while not user_string.lower() in ["","y","n","a","yes","no","all"]: + self.eprompt(message+" [Y/n/a]: ") + user_string = sys.stdin.readline().rstrip('\n') + user_string = user_string.strip() + if user_string.lower() in ["a","all"]: + self.options['accept_all'] = True + answer = user_string.lower() in ["","y","a","yes","all"] + return answer + + def progress_controller(self, size, key, clean_list, file_type): + """Callback function for doCleanup. It outputs data according to the + options configured. + Alternatively it handles user interaction for decisions that are + required. + + @param size: Integer of the file(s) size + @param key: the filename/pkgname currently being processed + @param clean_list: list of files being processed. + """ + if not self.options['quiet']: + # pretty print mode + print(self.prettySize(size,True), self.pkg_color(key)) + elif self.options['pretend'] or self.options['interactive']: + # file list mode + for file_ in clean_list: + print(file_) + if self.options['pretend']: + return False + elif not self.options['interactive'] \ + or self.options['accept_all'] \ + or self.yesNoAllPrompt("Do you want to delete this " + file_type + "?"): + return True + return False + + def total(self, mode, size, num_files, verb, action): + """outputs the formatted totals to stdout + + @param mode: sets color and message. 1 of ['normal', 'deprecated'] + @param size: total space savings + @param num_files: total number of files + @param verb: string eg. 1 of ["would be", "has been"] + @param action: string eg 1 of ['distfiles', 'packages'] + """ + self.set_colors(mode) + if mode =="normal": + message="Total space from "+red(str(num_files))+" files "+\ + verb+" freed in the " + action + " directory" + print( " ===========") + print( self.prettySize(size, True, red), message) + elif mode == "deprecated": + message = "Total space from "+red(str(num_files))+" package files\n"+\ + " Re-run the last command with the -D " +\ + "option to clean them as well" + print( " ===========") + print( self.prettySize(size, True, red), message) + + def list_pkgs(self, pkgs): + """outputs the packages to stdout + + @param pkgs: dict. of {cat/pkg-ver: src_uri,} + """ + indent = ' ' * 12 + for key in pkgs: + print( indent,self.pkg_color(key)) + print() diff --git a/pym/gentoolkit/eclean/pkgindex.py b/pym/gentoolkit/eclean/pkgindex.py new file mode 100644 index 0000000..f9d9f3c --- /dev/null +++ b/pym/gentoolkit/eclean/pkgindex.py @@ -0,0 +1,89 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + + +import subprocess +import sys + +import gentoolkit.pprinter as pp + +import portage +from portage import os + + +class PkgIndex(object): + """Handle the cleaning of the binpkg Package + Index file + + @type output: class + @param output: optional output class for printing + """ + + def __init__(self, controller=None): + self.controller = controller + + + def _get_emaint_binhost(self): + """Obtain a reference to the binhost module class + + @sets: self.binhost to BinhostHandler class + @rtype: boolean + """ + try: + self.emaint_control = Modules() + self.binhost = self.emaint_control._get_class('binhost') + except InvalidModuleName as er: + print( pp.error("Error importing emaint binhost module"), file=sys.stderr) + print( pp.error("Original error: " + er), file=sys.stderr) + except: + return False + return True + + + def _load_modules(self): + """Import the emaint modules and report the success/fail of them + """ + try: + from emaint.module import Modules + from emaint.main import TaskHandler + except ImportError as e: + return False + return True + + + def clean_pkgs_index(self,): + """This will clean the binpkgs packages index file""" + go = self._load_modules() + if go: + if self.get_emaint_binhost(): + self.taskmaster = TaskHandler(show_progress_bar=True) + tasks = [self.binhost] + self.taskmaster.run_tasks(tasks) + + + def call_emaint(self): + """Run the stand alone emaint script from + a subprocess call. + + @rtype: integer + @return: the difference in file size + """ + file_ = os.path.join(portage.settings['PKGDIR'], 'Packages') + statinfo = os.stat(file_) + size1 = statinfo.st_size + command = "emaint --fix binhost" + try: + retcode = subprocess.call(command, shell=True) + if retcode < 0: + print( pp.error("Child was terminated by signal" + str(-retcode)), file=sys.stderr) + except OSError as e: + print( pp.error("Execution failed:" + e), file=sys.stderr) + print() + statinfo = os.stat(file_) + clean_size = size1 - statinfo.st_size + self.controller(clean_size, "Packages Index", file_, "Index") + return clean_size diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py new file mode 100644 index 0000000..4c5b0ac --- /dev/null +++ b/pym/gentoolkit/eclean/search.py @@ -0,0 +1,520 @@ +#!/usr/bin/python + +# Copyright 2003-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + + +import re +import stat +import sys + +import portage +from portage import os + +import gentoolkit +import gentoolkit.pprinter as pp +from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand, + exclDictExpandPkgname, exclMatchFilename) +#from gentoolkit.package import Package +from gentoolkit.helpers import walk + + +# Misc. shortcuts to some portage stuff: +port_settings = portage.settings +pkgdir = port_settings["PKGDIR"] + +err = sys.stderr +deprecated_message=""""Deprecation Warning: Installed package: %s + Is no longer in the tree or an installed overlay""" +DEPRECATED = pp.warn(deprecated_message) + +debug_modules = [] + + +def dprint(module, message): + if module in debug_modules: + print(message) + + +def get_distdir(): + """Returns DISTDIR if sane, else barfs.""" + + d = portage.settings["DISTDIR"] + if not os.path.isdir(d): + e = pp.error("%s does not appear to be a directory.\n" % d) + e += pp.error("Please set DISTDIR to a sane value.\n") + e += pp.error("(Check your /etc/make.conf and environment).") + print( e, file=sys.stderr) + exit(1) + return d + +distdir = get_distdir() + + +class DistfilesSearch(object): + """ + + @param output: verbose output method or (lambda x: None) to turn off + @param vardb: defaults to portage.db[portage.root]["vartree"].dbapi + is overridden for testing. + @param portdb: defaults to portage.portdb and is overriden for testing. +""" + + def __init__(self, + output, + portdb=portage.portdb, + vardb=portage.db[portage.root]["vartree"].dbapi, + ): + self.vardb =vardb + self.portdb = portdb + self.output = output + + def findDistfiles(self, + exclude={}, + destructive=False, + fetch_restricted=False, + package_names=False, + time_limit=0, + size_limit=0, + _distdir=distdir, + deprecate=False + ): + """Find all obsolete distfiles. + + XXX: what about cvs ebuilds? + I should install some to see where it goes... + + @param exclude: an exclusion dict as defined in + exclude.parseExcludeFile class. + @param destructive: boolean, defaults to False + @param fetch_restricted: boolean, defaults to False + @param package_names: boolean, defaults to False. + @param time_limit: integer time value as returned by parseTime() + @param size_limit: integer value of max. file size to keep or 0 to ignore. + @param _distdir: path to the distfiles dir being checked, defaults to portage. + @param deprecate: bool to control checking the clean dict. files for exclusion + + @rtype: dict + @return dict. of package files to clean i.e. {'cat/pkg-ver.tbz2': [filename],} + """ + clean_me = {} + pkgs = {} + saved = {} + deprecated = {} + installed_included = False + # create a big CPV->SRC_URI dict of packages + # whose distfiles should be kept + if (not destructive) or fetch_restricted: + self.output("...non-destructive type search") + # TODO fix fetch_restricted to save the installed packges filenames while processing + pkgs, _deprecated = self._non_destructive(destructive, fetch_restricted, exclude=exclude) + deprecated.update(_deprecated) + installed_included = True + if destructive: + self.output("...destructive type search: %d packages already found" %len(pkgs)) + pkgs, _deprecated = self._destructive(package_names, + exclude, pkgs, installed_included) + deprecated.update(_deprecated) + # gather the files to be cleaned + self.output("...checking limits for %d ebuild sources" + %len(pkgs)) + clean_me = self._check_limits(_distdir, + size_limit, time_limit, exclude) + # remove any protected files from the list + self.output("...removing protected sources from %s candidates to clean" + %len(clean_me)) + clean_me = self._remove_protected(pkgs, clean_me) + if not deprecate and len(exclude) and len(clean_me): + self.output("...checking final for exclusion from " +\ + "%s remaining candidates to clean" %len(clean_me)) + clean_me, saved = self._check_excludes(exclude, clean_me) + return clean_me, saved, deprecated + + +####################### begin _check_limits code block + + def _check_limits(self, + _distdir, + size_limit, + time_limit, + exclude, + clean_me={} + ): + """Checks files if they exceed size and/or time_limits, etc. + """ + checks = [self._isreg_limit_] + if size_limit: + checks.append(self._size_limit_) + self.size_limit = size_limit + else: + self.output(" - skipping size limit check") + if time_limit: + checks.append(self._time_limit_) + self.time_limit = time_limit + else: + self.output(" - skipping time limit check") + if 'filenames' in exclude: + checks.append(self._filenames_limit_) + self.exclude = exclude + else: + self.output(" - skipping exclude filenames check") + max_index = len(checks) + for file in os.listdir(_distdir): + filepath = os.path.join(_distdir, file) + try: + file_stat = os.stat(filepath) + except: + continue + _index = 0 + next = True + skip_file = False + while _index<max_index and next: + next, skip_file = checks[_index](file_stat, file) + _index +=1 + if skip_file: + continue + # this is a candidate for cleaning + #print( "Adding file to clean_list:", file) + clean_me[file]=[filepath] + return clean_me + + def _isreg_limit_(self, file_stat, file): + """check if file is a regular file.""" + is_reg_file = stat.S_ISREG(file_stat[stat.ST_MODE]) + return is_reg_file, not is_reg_file + + def _size_limit_(self, file_stat, file): + """checks if the file size exceeds the size_limit""" + if (file_stat[stat.ST_SIZE] >= self.size_limit): + #print( "size match ", file, file_stat[stat.ST_SIZE]) + return False, True + return True, False + + def _time_limit_(self, file_stat, file): + """checks if the file exceeds the time_limit""" + if (file_stat[stat.ST_MTIME] >= self.time_limit): + #print( "time match ", file, file_stat[stat.ST_MTIME]) + return False, True + return True,False + + def _filenames_limit_(self, file_stat, file): + """checks if the file matches an exclusion file listing""" + # Try to match file name directly + if file in self.exclude['filenames']: + return False, True + # See if file matches via regular expression matching + else: + file_match = False + for file_entry in self.exclude['filenames']: + if self.exclude['filenames'][file_entry].match(file): + file_match = True + break + if file_match: + return False, True + return True, False + +####################### end _check_limits code block + + def _remove_protected(self, + pkgs, + clean_me + ): + """Remove files owned by some protected packages. + + @returns packages to clean + @rtype: dictionary + """ + # this regexp extracts files names from SRC_URI. It is not very precise, + # but we don't care (may return empty strings, etc.), since it is fast. + file_regexp = re.compile(r'([a-zA-Z0-9_,\.\-\+\~]*)[\s\)]') + for cpv in pkgs: + for file in file_regexp.findall(pkgs[cpv]+"\n"): + if file in clean_me: + del clean_me[file] + # no need to waste IO time if there is nothing left to clean + if not len(clean_me): + return clean_me + return clean_me + + def _non_destructive(self, + destructive, + fetch_restricted, + pkgs_ = {}, + exclude={} + ): + """performs the non-destructive checks + + @param destructive: boolean + @param pkgs_: starting dictionary to add to + defaults to {}. + + @returns packages and thier SRC_URI's: {cpv: src_uri,} + @rtype: dictionary + """ + pkgs = pkgs_.copy() + deprecated = {} + # the following code block was split to optimize for speed + # list all CPV from portree (yeah, that takes time...) + self.output(" - getting complete ebuild list") + cpvs = set(self.portdb.cpv_all()) + # now add any installed cpv's that are not in the tree or overlays + installed_cpvs = self.vardb.cpv_all() + cpvs.update(installed_cpvs) + if fetch_restricted and destructive: + self.output(" - getting source file names " + + "for %d installed ebuilds" %len(installed_cpvs)) + pkgs, _deprecated = self._unrestricted(pkgs, installed_cpvs) + deprecated.update(_deprecated) + # remove the installed cpvs then check the remaining for fetch restiction + cpvs.difference_update(installed_cpvs) + self.output(" - getting fetch-restricted source file names " + + "for %d remaining ebuilds" %len(cpvs)) + pkgs, _deprecated = self._fetch_restricted(destructive, pkgs, cpvs) + deprecated.update(_deprecated) + else: + self.output(" - getting source file names " + + "for %d ebuilds" %len(cpvs)) + pkgs, _deprecated = self._unrestricted(pkgs, cpvs) + deprecated.update(_deprecated) + return pkgs, deprecated + + def _fetch_restricted(self, destructive, pkgs_, cpvs): + """perform fetch restricted non-destructive source + filename lookups + + @param destructive: boolean + @param pkgs_: starting dictionary to add to + @param cpvs: set of (cat/pkg-ver, ...) identifiers + + @return a new pkg dictionary + @rtype: dictionary + """ + pkgs = pkgs_.copy() + deprecated = {} + for cpv in cpvs: + # get SRC_URI and RESTRICT from aux_get + try: # main portdb + (src_uri,restrict) = \ + self.portdb.aux_get(cpv,["SRC_URI","RESTRICT"]) + # keep fetch-restricted check + # inside try so it is bypassed on KeyError + if 'fetch' in restrict: + pkgs[cpv] = src_uri + except KeyError: + try: # installed vardb + (src_uri,restrict) = \ + self.vardb.aux_get(cpv,["SRC_URI","RESTRICT"]) + deprecated[cpv] = src_uri + self.output(DEPRECATED %cpv) + # keep fetch-restricted check + # inside try so it is bypassed on KeyError + if 'fetch' in restrict: + pkgs[cpv] = src_uri + except KeyError: + self.output(" - Key Error looking up: " + cpv) + return pkgs, deprecated + + def _unrestricted(self, pkgs_, cpvs): + """Perform unrestricted source filenames lookups + + @param pkgs_: starting packages dictionary + @param cpvs: set of (cat/pkg-ver, ...) identifiers + + @return a new pkg dictionary + @rtype: dictionary + """ + pkgs = pkgs_.copy() + deprecated = {} + for cpv in cpvs: + # get SRC_URI from aux_get + try: + pkgs[cpv] = self.portdb.aux_get(cpv,["SRC_URI"])[0] + except KeyError: + try: # installed vardb + pkgs[cpv] = self.vardb.aux_get(cpv,["SRC_URI"])[0] + deprecated[cpv] = pkgs[cpv] + self.output(DEPRECATED %cpv) + except KeyError: + self.output(" - Key Error looking up: " + cpv) + return pkgs, deprecated + + def _destructive(self, + package_names, + exclude, + pkgs_={}, + installed_included=False + ): + """Builds on pkgs according to input options + + @param package_names: boolean + @param exclude: an exclusion dict as defined in + exclude.parseExcludeFile class. + @param pkgs: starting dictionary to add to + defaults to {}. + @param installed_included: bool. pkgs already + has the installed cpv's added. + + @returns pkgs: {cpv: src_uri,} + """ + pkgs = pkgs_.copy() + deprecated = {} + pkgset = set() + if not installed_included: + if not package_names: + # list all installed CPV's from vartree + #print( "_destructive: getting vardb.cpv_all") + pkgset.update(self.vardb.cpv_all()) + self.output(" - processing %s installed ebuilds" % len(pkgset)) + elif package_names: + # list all CPV's from portree for CP's in vartree + #print( "_destructive: getting vardb.cp_all") + cps = self.vardb.cp_all() + self.output(" - processing %s installed packages" % len(cps)) + for package in cps: + pkgset.update(self.portdb.cp_list(package)) + self.output(" - processing excluded") + excludes = self._get_excludes(exclude) + excludes_length = len(excludes) + pkgset.update(excludes) + pkgs_done = set(list(pkgs)) + pkgset.difference_update(pkgs_done) + self.output( + " - (%d of %d total) additional excluded packages to get source filenames for" + %(len(pkgset), excludes_length)) + #self.output(" - processing %d ebuilds for filenames" %len(pkgset)) + pkgs, _deprecated = self._unrestricted(pkgs, pkgset) + deprecated.update(_deprecated) + #self.output(" - done...") + return pkgs, deprecated + + def _get_excludes(self, exclude): + """Expands the exclude dictionary into a set of + CPV's + + @param exclude: dictionary of exclusion categories, + packages to exclude from the cleaning + + @rtype: set + @return set of package cpv's + """ + pkgset = set() + for cp in exclDictExpand(exclude): + # add packages from the exclude file + pkgset.update(self.portdb.cp_list(cp)) + return pkgset + + def _check_excludes(self, exclude, clean_me): + """Performs a last minute check on remaining filenames + to see if they should be protected. Since if the pkg-version + was deprecated it would not have been matched to a + source filename and removed. + + @param exclude: an exclusion dictionary + @param clean_me: the list of filenames for cleaning + + @rtype: dict of packages to clean + """ + saved = {} + pn_excludes = exclDictExpandPkgname(exclude) + dprint("excludes", "_check_excludes: made it here ;)") + if not pn_excludes: + return clean_me, saved + dprint("excludes", pn_excludes) + for key in list(clean_me): + if exclMatchFilename(pn_excludes, key): + saved[key] = clean_me[key] + del clean_me[key] + self.output(" ...Saved excluded package filename: " + key) + return clean_me, saved + + +def findPackages( + options, + exclude={}, + destructive=False, + time_limit=0, + package_names=False, + pkgdir=None, + port_dbapi=portage.db[portage.root]["porttree"].dbapi, + var_dbapi=portage.db[portage.root]["vartree"].dbapi + ): + """Find all obsolete binary packages. + + XXX: packages are found only by symlinks. + Maybe i should also return .tbz2 files from All/ that have + no corresponding symlinks. + + @param options: dict of options determined at runtime + @param exclude: an exclusion dict as defined in + exclude.parseExcludeFile class. + @param destructive: boolean, defaults to False + @param time_limit: integer time value as returned by parseTime() + @param package_names: boolean, defaults to False. + used only if destructive=True + @param pkgdir: path to the binary package dir being checked + @param port_dbapi: defaults to portage.db[portage.root]["porttree"].dbapi + can be overridden for tests. + @param var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi + can be overridden for tests. + + @rtype: dict + @return clean_me i.e. {'cat/pkg-ver.tbz2': [filepath],} + """ + clean_me = {} + # create a full package dictionary + + # now do an access test, os.walk does not error for "no read permission" + try: + test = os.listdir(pkgdir) + del test + except EnvironmentError as er: + print( pp.error("Error accessing PKGDIR." ), file=sys.stderr) + print( pp.error("(Check your /etc/make.conf and environment)."), file=sys.stderr) + print( pp.error("Error: %s" %str(er)), file=sys.stderr) + exit(1) + for root, dirs, files in walk(pkgdir): + if root[-3:] == 'All': + continue + for file in files: + if not file[-5:] == ".tbz2": + # ignore non-tbz2 files + continue + path = os.path.join(root, file) + category = os.path.split(root)[-1] + cpv = category+"/"+file[:-5] + st = os.lstat(path) + if time_limit and (st[stat.ST_MTIME] >= time_limit): + # time-limit exclusion + continue + # dict is cpv->[files] (2 files in general, because of symlink) + clean_me[cpv] = [path] + #if os.path.islink(path): + if stat.S_ISLNK(st[stat.ST_MODE]): + clean_me[cpv].append(os.path.realpath(path)) + # keep only obsolete ones + if destructive: + dbapi = var_dbapi + if package_names: + cp_all = dict.fromkeys(dbapi.cp_all()) + else: + cp_all = {} + else: + dbapi = port_dbapi + cp_all = {} + for cpv in list(clean_me): + if exclDictMatchCP(exclude,portage.cpv_getkey(cpv)): + # exclusion because of the exclude file + del clean_me[cpv] + continue + if dbapi.cpv_exists(cpv): + # exclusion because pkg still exists (in porttree or vartree) + del clean_me[cpv] + continue + if portage.cpv_getkey(cpv) in cp_all: + # exlusion because of --package-names + del clean_me[cpv] + + return clean_me |